Support arbitrary PEP-517 build systems. (pantsbuild#13285)

If we find a pyproject.toml in the chroot, containing a [build-system] table, we
run its build_backend in an environment containing its requires.

Otherwise we fall back to the legacy setuptools behavior.

[ci skip-rust]

[ci skip-build-wheels]

src/python/pants/backend/python/goals/setup_py.py

@@ -15,7 +15,7 @@
 from typing import Any, DefaultDict, Dict, List, Mapping, Tuple, cast
 
 from pants.backend.python.macros.python_artifact import PythonArtifact
-from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet, Setuptools
+from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet
 from pants.backend.python.target_types import (
     GenerateSetupField,
     PythonDistributionEntryPointsField,
@@ -31,7 +31,13 @@
     WheelConfigSettingsField,
     WheelField,
 )
-from pants.backend.python.util_rules.dists import DistBuildRequest, DistBuildResult, distutils_repr
+from pants.backend.python.util_rules.dists import (
+    BuildSystem,
+    BuildSystemRequest,
+    DistBuildRequest,
+    DistBuildResult,
+    distutils_repr,
+)
 from pants.backend.python.util_rules.dists import rules as dists_rules
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import PexRequirements
@@ -173,8 +179,8 @@ class ExportedTargetRequirements(DeduplicatedCollection[str]):
 
 
 @dataclass(frozen=True)
-class SetupPySources:
-    """The sources required by a setup.py command.
+class DistBuildSources:
+    """The sources required to build a distribution.
 
     Includes some information derived from analyzing the source, namely the packages, namespace
     packages and resource files in the source.
@@ -187,8 +193,8 @@ class SetupPySources:
 
 
 @dataclass(frozen=True)
-class SetupPyChrootRequest:
-    """A request to create a chroot containing a setup.py and the sources it operates on."""
+class DistBuildChrootRequest:
+    """A request to create a chroot for building a dist in."""
 
     exported_target: ExportedTarget
     py2: bool  # Whether to use py2 or py3 package semantics.
@@ -367,16 +373,21 @@ def validate_commands(commands: tuple[str, ...]):
 
 @rule
 async def package_python_dist(
-    field_set: PythonDistributionFieldSet, python_setup: PythonSetup, setuptools: Setuptools
+    field_set: PythonDistributionFieldSet,
+    python_setup: PythonSetup,
 ) -> BuiltPackage:
     transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
     exported_target = ExportedTarget(transitive_targets.roots[0])
+
     interpreter_constraints = InterpreterConstraints.create_from_targets(
         transitive_targets.closure, python_setup
     ) or InterpreterConstraints(python_setup.interpreter_constraints)
     chroot = await Get(
         DistBuildChroot,
-        SetupPyChrootRequest(exported_target, py2=interpreter_constraints.includes_python2()),
+        DistBuildChrootRequest(
+            exported_target,
+            py2=interpreter_constraints.includes_python2(),
+        ),
     )
 
     dist_tgt = exported_target.target
@@ -399,11 +410,13 @@ async def package_python_dist(
         chroot_prefix = "chroot"
         working_directory = os.path.join(chroot_prefix, chroot.working_directory)
         prefixed_chroot = await Get(Digest, AddPrefix(chroot.digest, chroot_prefix))
+        build_system = await Get(
+            BuildSystem, BuildSystemRequest(prefixed_chroot, working_directory)
+        )
         setup_py_result = await Get(
             DistBuildResult,
             DistBuildRequest(
-                requires=setuptools.pex_requirements(),
-                build_backend="setuptools.build_meta",
+                build_system=build_system,
                 interpreter_constraints=interpreter_constraints,
                 build_wheel=wheel,
                 build_sdist=sdist,
@@ -504,7 +517,7 @@ async def determine_setup_kwargs(
 @dataclass(frozen=True)
 class GenerateSetupPyRequest:
     exported_target: ExportedTarget
-    sources: SetupPySources
+    sources: DistBuildSources
 
 
 @dataclass(frozen=True)
@@ -514,7 +527,7 @@ class GeneratedSetupPy:
 
 @rule
 async def generate_chroot(
-    request: SetupPyChrootRequest, subsys: SetupPyGeneration
+    request: DistBuildChrootRequest, subsys: SetupPyGeneration
 ) -> DistBuildChroot:
     generate_setup = request.exported_target.target.get(GenerateSetupField).value
     if generate_setup is None:
@@ -535,7 +548,7 @@ async def generate_chroot(
         )
     # ... to here.
 
-    sources = await Get(SetupPySources, SetupPyChrootRequest, request)
+    sources = await Get(DistBuildSources, DistBuildChrootRequest, request)
 
     if generate_setup:
         generated_setup_py = await Get(
@@ -659,8 +672,8 @@ def _format_entry_points(
 
 @rule
 async def get_sources(
-    request: SetupPyChrootRequest, union_membership: UnionMembership
-) -> SetupPySources:
+    request: DistBuildChrootRequest, union_membership: UnionMembership
+) -> DistBuildSources:
     owned_deps, transitive_targets = await MultiGet(
         Get(OwnedDependencies, DependencyOwner(request.exported_target)),
         Get(
@@ -704,7 +717,7 @@ async def get_sources(
         init_py_digest_contents=init_py_digest_contents,
         py2=request.py2,
     )
-    return SetupPySources(
+    return DistBuildSources(
         digest=all_sources.stripped_source_files.snapshot.digest,
         packages=packages,
         namespace_packages=namespace_packages,

src/python/pants/backend/python/goals/setup_py_test.py

@@ -13,6 +13,8 @@
     AmbiguousOwnerError,
     DependencyOwner,
     DistBuildChroot,
+    DistBuildChrootRequest,
+    DistBuildSources,
     ExportedTarget,
     ExportedTargetRequirements,
     FinalizedSetupKwargs,
@@ -25,9 +27,7 @@
     OwnedDependency,
     SetupKwargs,
     SetupKwargsRequest,
-    SetupPyChrootRequest,
     SetupPyGeneration,
-    SetupPySources,
     declares_pkg_resources_namespace_package,
     determine_setup_kwargs,
     generate_chroot,
@@ -109,8 +109,8 @@ def chroot_rule_runner() -> RuleRunner:
             setup_kwargs_plugin,
             SubsystemRule(SetupPyGeneration),
             UnionRule(SetupKwargsRequest, PluginSetupKwargsRequest),
-            QueryRule(DistBuildChroot, (SetupPyChrootRequest,)),
-            QueryRule(SetupPySources, (SetupPyChrootRequest,)),
+            QueryRule(DistBuildChroot, (DistBuildChrootRequest,)),
+            QueryRule(DistBuildSources, (DistBuildChrootRequest,)),
             QueryRule(FinalizedSetupKwargs, (GenerateSetupPyRequest,)),
         ]
     )
@@ -123,13 +123,13 @@ def assert_chroot(
     addr: Address,
 ) -> None:
     tgt = rule_runner.get_target(addr)
-    req = SetupPyChrootRequest(ExportedTarget(tgt), py2=False)
+    req = DistBuildChrootRequest(ExportedTarget(tgt), py2=False)
     chroot = rule_runner.request(DistBuildChroot, [req])
     snapshot = rule_runner.request(Snapshot, [chroot.digest])
     assert sorted(expected_files) == sorted(snapshot.files)
 
     if expected_setup_kwargs is not None:
-        sources = rule_runner.request(SetupPySources, [req])
+        sources = rule_runner.request(DistBuildSources, [req])
         setup_kwargs = rule_runner.request(
             FinalizedSetupKwargs, [GenerateSetupPyRequest(ExportedTarget(tgt), sources)]
         )
@@ -141,7 +141,7 @@ def assert_chroot_error(rule_runner: RuleRunner, addr: Address, exc_cls: type[Ex
     with pytest.raises(ExecutionError) as excinfo:
         rule_runner.request(
             DistBuildChroot,
-            [SetupPyChrootRequest(ExportedTarget(tgt), py2=False)],
+            [DistBuildChrootRequest(ExportedTarget(tgt), py2=False)],
         )
     ex = excinfo.value
     assert len(ex.wrapped_exceptions) == 1
@@ -566,7 +566,7 @@ def assert_sources(
                 *target_types_rules.rules(),
                 *python_sources.rules(),
                 QueryRule(OwnedDependencies, (DependencyOwner,)),
-                QueryRule(SetupPySources, (SetupPyChrootRequest,)),
+                QueryRule(DistBuildSources, (DistBuildChrootRequest,)),
             ]
         )
 
@@ -604,8 +604,8 @@ def assert_sources(
         )
         owner_tgt = rule_runner.get_target(Address("src/python/foo", target_name="dist"))
         srcs = rule_runner.request(
-            SetupPySources,
-            [SetupPyChrootRequest(ExportedTarget(owner_tgt), py2=False)],
+            DistBuildSources,
+            [DistBuildChrootRequest(ExportedTarget(owner_tgt), py2=False)],
         )
         chroot_snapshot = rule_runner.request(Snapshot, [srcs.digest])
 

src/python/pants/backend/python/util_rules/dists.py

@@ -7,7 +7,11 @@
 import os
 from collections import abc
 from dataclasses import dataclass
+from typing import Any, Mapping
 
+import toml
+
+from pants.backend.python.subsystems.setuptools import Setuptools
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import (
     Lockfile,
@@ -18,7 +22,18 @@
     VenvPexProcess,
 )
 from pants.backend.python.util_rules.pex import rules as pex_rules
-from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests, RemovePrefix, Snapshot
+from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
+from pants.engine.fs import (
+    CreateDigest,
+    Digest,
+    DigestContents,
+    DigestSubset,
+    FileContent,
+    MergeDigests,
+    PathGlobs,
+    RemovePrefix,
+    Snapshot,
+)
 from pants.engine.internals.selectors import Get
 from pants.engine.process import ProcessResult
 from pants.engine.rules import collect_rules, rule
@@ -38,15 +53,68 @@ class InvalidBuildConfigError(Exception):
 
 
 @dataclass(frozen=True)
-class DistBuildRequest:
-    """A request to build dists via a PEP 517 build backend."""
+class BuildSystemRequest:
+    """A request to find build system config in the given dir of the given digest."""
+
+    digest: Digest
+    working_directory: str
 
-    # TODO: Create one of these from information in pyproject.toml.
 
-    # PEP-517/518 fields.
+@dataclass(frozen=True)
+class BuildSystem:
+    """A PEP 517/518 build system configuration."""
+
     requires: PexRequirements | Lockfile | LockfileContent
     build_backend: str
 
+    @classmethod
+    def legacy(cls, setuptools: Setuptools) -> BuildSystem:
+        return cls(setuptools.pex_requirements(), "setuptools.build_meta:__legacy__")
+
+
+@rule
+async def find_build_system(request: BuildSystemRequest, setuptools: Setuptools) -> BuildSystem:
+    digest_contents = await Get(
+        DigestContents,
+        DigestSubset(
+            request.digest,
+            PathGlobs(
+                globs=[os.path.join(request.working_directory, "pyproject.toml")],
+                glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
+            ),
+        ),
+    )
+    ret = None
+    if digest_contents:
+        file_content = next(iter(digest_contents))
+        settings: Mapping[str, Any] = toml.loads(file_content.content.decode())
+        build_system = settings.get("build-system")
+        if build_system is not None:
+            build_backend = build_system.get("build-backend")
+            if build_backend is None:
+                raise InvalidBuildConfigError(
+                    f"No build-backend found in the [build-system] table in {file_content.path}"
+                )
+            requires = build_system.get("requires")
+            if requires is None:
+                raise InvalidBuildConfigError(
+                    f"No requires found in the [build-system] table in {file_content.path}"
+                )
+            ret = BuildSystem(PexRequirements(requires), build_backend)
+    # Per PEP 517: "If the pyproject.toml file is absent, or the build-backend key is missing,
+    #   the source tree is not using this specification, and tools should revert to the legacy
+    #   behaviour of running setup.py."
+    if ret is None:
+        ret = BuildSystem.legacy(setuptools)
+    return ret
+
+
+@dataclass(frozen=True)
+class DistBuildRequest:
+    """A request to build dists via a PEP 517 build backend."""
+
+    build_system: BuildSystem
+
     # TODO: Support backend_path (https://www.python.org/dev/peps/pep-0517/#in-tree-build-backends)
 
     interpreter_constraints: InterpreterConstraints
@@ -100,7 +168,7 @@ class DistBuildResult:
 
 def interpolate_backend_shim(dist_dir: str, request: DistBuildRequest) -> bytes:
     # See https://www.python.org/dev/peps/pep-0517/#source-trees.
-    module_path, _, object_path = request.build_backend.partition(":")
+    module_path, _, object_path = request.build_system.build_backend.partition(":")
     backend_object = f"{module_path}.{object_path}" if object_path else module_path
 
     def config_settings_repr(cs: FrozenDict[str, tuple[str, ...]] | None) -> str:
@@ -128,7 +196,7 @@ async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup)
         PexRequest(
             output_filename="build_backend.pex",
             internal_only=True,
-            requirements=request.requires,
+            requirements=request.build_system.requires,
             interpreter_constraints=request.interpreter_constraints,
         ),
     )
@@ -162,9 +230,9 @@ async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup)
             working_directory=request.working_directory,
             output_directories=(dist_dir,),  # Relative to the working_directory.
             description=(
-                f"Run {request.build_backend} for {request.target_address_spec}"
+                f"Run {request.build_system.build_backend} for {request.target_address_spec}"
                 if request.target_address_spec
-                else f"Run {request.build_backend}"
+                else f"Run {request.build_system.build_backend}"
             ),
             level=LogLevel.DEBUG,
         ),
@@ -181,7 +249,7 @@ async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup)
     for dist_type, path in paths.items():
         if path not in output_snapshot.files:
             raise BuildBackendError(
-                f"Build backend {request.build_backend} did not create "
+                f"Build backend {request.build_system.build_backend} did not create "
                 f"expected {dist_type} file {path}"
             )
     return DistBuildResult(

testprojects/src/python/native/BUILD

@@ -8,9 +8,11 @@ python_sources(
 
 resources(name="impl", sources=["*.c"])
 
+resources(name="pyproject", sources=["pyproject.toml"])
+
 python_distribution(
     name = "dist",
-    dependencies = [":impl", ":lib"],
+    dependencies = [":pyproject", ":impl", ":lib"],
     generate_setup = False,
     provides = python_artifact(
         name = "native",

testprojects/src/python/native/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+
+requires = [
+    "setuptools==57.5.0",
+    "wheel==0.37.0",
+    "ansicolors==1.1.8",
+]
+build-backend = "setuptools.build_meta"

testprojects/src/python/native/setup.py

@@ -1,7 +1,7 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-
+from colors import red
 from setuptools import Extension, setup  # type: ignore[import]
 
 native_impl = Extension("native.impl", sources=["impl.c"])
@@ -13,4 +13,5 @@
     namespace_packages=["native"],
     package_dir={"native": "."},
     ext_modules=[native_impl],
+    description=red("Proof that custom PEP-517 build-time requirements work"),
 )