Allow python tools to be installed from a user resolve. (pantsbuild#1…

…8418)

This is intended to eventually replace the "tool lockfile"
functionality, so that we have just one uniform way of generating
lockfiles.

src/python/pants/backend/python/pip_requirement.py

@@ -13,14 +13,7 @@
 
 
 class PipRequirement:
-    """A Pip-style requirement.
-
-    Currently just a drop-in replacement for pkg_resources.Requirement.
-
-    TODO:  Once this class has fully replaced relevant uses of pkg_resources.Requirement,
-      we will enhance this class to support Pip requirements that are not parseable by
-      pkg_resources.Requirement, such as old-style VCS requirements, --hash option suffixes etc.
-    """
+    """A Pip-style requirement."""
 
     @classmethod
     def parse(cls, line: str, description_of_origin: str = "") -> PipRequirement:

src/python/pants/backend/python/subsystems/python_tool_base.py

@@ -14,6 +14,7 @@
     LoadedLockfileRequest,
     Lockfile,
     PexRequirements,
+    Resolve,
 )
 from pants.core.goals.generate_lockfiles import DEFAULT_TOOL_LOCKFILE, NO_TOOL_LOCKFILE
 from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest
@@ -22,7 +23,7 @@
 from pants.option.errors import OptionsError
 from pants.option.option_types import BoolOption, StrListOption, StrOption
 from pants.option.subsystem import Subsystem
-from pants.util.docutil import bin_name
+from pants.util.docutil import bin_name, doc_url
 from pants.util.strutil import softwrap
 
 
@@ -34,11 +35,16 @@ class PythonToolRequirementsBase(Subsystem):
     # Subclasses do not need to override.
     default_extra_requirements: ClassVar[Sequence[str]] = []
 
+    # Subclasses may set to override the value computed from default_version and
+    # default_extra_requirements.
+    # TODO: Once we get rid of those options, subclasses must set this to loose
+    #  requirements that reflect any minimum capabilities Pants assumes about the tool.
+    default_requirements: Sequence[str] = []
+
     default_interpreter_constraints: ClassVar[Sequence[str]] = []
     register_interpreter_constraints: ClassVar[bool] = False
 
-    # If this tool does not mix with user requirements (e.g. Flake8 and Isort, but not Pylint and
-    # Pytest), you should set this to True.
+    # If this tool does not mix with user requirements you should set this to True.
     #
     # You also need to subclass `GeneratePythonToolLockfileSentinel` and create a rule that goes
     # from it -> GeneratePythonLockfile by calling `GeneratePythonLockfile.from_python_tool()`.
@@ -47,6 +53,41 @@ class PythonToolRequirementsBase(Subsystem):
     default_lockfile_resource: ClassVar[tuple[str, str] | None] = None
     default_lockfile_url: ClassVar[str | None] = None
 
+    install_from_resolve = StrOption(
+        advanced=True,
+        default=None,
+        help=lambda cls: softwrap(
+            f"""\
+            If specified, install the tool using the lockfile for this named resolve.
+
+            This resolve must be defined in [python].resolves, as described in
+            {doc_url("python-third-party-dependencies#user-lockfiles")}, and its lockfile must
+            provide the requirements named in the `requirements` option.
+
+            If unspecified, and the `lockfile` option is unset, the tool will be installed
+            using the default lockfile shipped with Pants.
+
+            If unspecified, and the `lockfile` option is set, the tool will use the custom
+            `{cls.options_scope}` "tool lockfile" generated from the `version` and
+            `extra_requirements` options. But note that this mechanism will soon be deprecated.
+            """
+        ),
+    )
+    # TODO: After we deprecate and remove the tool lockfile concept, we can remove the
+    #  version and extra_requirements options and directly list loosely-constrained
+    #  requirements for each tool in this option's default. The specific versions will then
+    #  come either from the default lockfile we provide, or from a user lockfile.
+    requirements = StrListOption(
+        advanced=True,
+        default=lambda cls: cls.default_requirements
+        or sorted([cls.default_version, *cls.default_extra_requirements]),
+        help=lambda cls: softwrap(
+            """\
+            If install_from_resolve is specified, it will install these distribution packages,
+            using the versions from the specified resolve.
+            """
+        ),
+    )
     version = StrOption(
         advanced=True,
         default=lambda cls: cls.default_version,
@@ -147,6 +188,11 @@ def pex_requirements(
         if not self.uses_lockfile:
             return PexRequirements(requirements)
 
+        if self.install_from_resolve:
+            return PexRequirements(
+                self.requirements, from_superset=Resolve(self.install_from_resolve)
+            )
+
         hex_digest = calculate_invalidation_digest(requirements)
 
         if self.lockfile == DEFAULT_TOOL_LOCKFILE:

src/python/pants/backend/python/subsystems/python_tool_base_test.py

@@ -0,0 +1,31 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.util_rules.pex_requirements import PexRequirements, Resolve
+from pants.testutil.option_util import create_subsystem
+from pants.util.ordered_set import FrozenOrderedSet
+
+
+class _DummyTool(PythonToolBase):
+    options_scope = "dummy"
+    register_lockfile = True
+    default_lockfile_resource = ("dummy", "dummy")
+    default_lockfile_url = "dummy"
+
+
+def test_install_from_resolve_default() -> None:
+    tool = create_subsystem(
+        _DummyTool,
+        lockfile="dummy.lock",
+        install_from_resolve="dummy_resolve",
+        requirements=["foo", "bar", "baz"],
+        version="",
+        extra_requirements=[],
+    )
+    pex_reqs = tool.pex_requirements()
+    assert isinstance(pex_reqs, PexRequirements)
+    assert pex_reqs.from_superset == Resolve("dummy_resolve")
+    assert pex_reqs.req_strings == FrozenOrderedSet(["bar", "baz", "foo"])

src/python/pants/backend/python/subsystems/setuptools_scm.py

@@ -22,6 +22,7 @@ class SetuptoolsSCM(PythonToolBase):
 
     default_version = "setuptools-scm==6.4.2"
     default_main = EntryPoint("setuptools_scm")
+    default_requirements = ["setuptools-scm"]
 
     register_interpreter_constraints = True
     default_interpreter_constraints = ["CPython>=3.7,<4"]

src/python/pants/backend/python/util_rules/pex_requirements.py

@@ -502,6 +502,8 @@ def validate_metadata(
         metadata=metadata,
         validation=validation,
         lockfile=lockfile,
+        is_old_style_tool_lockfile=lockfile.resolve_name not in python_setup.resolves,
+        is_default_user_lockfile=lockfile.resolve_name == python_setup.default_resolve,
         user_interpreter_constraints=interpreter_constraints,
         user_requirements=user_requirements,
         maybe_constraints_file_path=(
@@ -557,12 +559,14 @@ def _invalid_lockfile_error(
     validation: LockfileMetadataValidation,
     lockfile: Lockfile,
     *,
+    is_old_style_tool_lockfile: bool,
+    is_default_user_lockfile: bool,
     user_requirements: set[PipRequirement],
     user_interpreter_constraints: InterpreterConstraints,
     maybe_constraints_file_path: str | None,
 ) -> Iterator[str]:
     resolve = lockfile.resolve_name
-    yield "You are using "
+    yield "\n\nYou are using "
     if lockfile.url.startswith("resource://"):
         yield f"the built-in `{resolve}` lockfile provided by Pants "
     else:
@@ -578,58 +582,85 @@ def _invalid_lockfile_error(
         for i in validation.failure_reasons
     ):
         yield softwrap(
-            f"""
+            """
             - The lockfile does not provide all the necessary requirements. You must
-            modify the input requirements and/or regenerate the lockfile (see below)`.
-
-            If `{resolve}` is a Python tool, the necessary requirements are specified by
-            `[{resolve}].version`, `[{resolve}].extra_requirements`, and/or
-            `[{resolve}].source_plugins`, and the custom lockfile destination is specified by
-            `[{resolve}].lockfile`.
-
-            Otherwise, the necessary requirements are specified by your code's dependencies,
-            and the lockfile destination is specified by `[python].resolves`.
-
-            See {doc_url('python-third-party-dependencies')} for details.
+            modify the input requirements and/or regenerate the lockfile (see below).
             """
         ) + "\n\n"
+        if is_old_style_tool_lockfile:
+            yield softwrap(
+                f"""
+                - The necessary requirements are specified by `[{resolve}].version`,
+                `[{resolve}].extra_requirements`, and/or `[{resolve}].source_plugins`.
+
+                - The custom lockfile destination is specified by `[{resolve}].lockfile`.
+                """
+            )
+        elif is_default_user_lockfile:
+            yield softwrap(
+                f"""
+                - The necessary requirements are specified by requirements targets marked with
+                `resolve="{resolve}"`, or those with no explicit resolve (since `{resolve}` is the
+                default for this repo).
+
+                - The lockfile destination is specified by the `{resolve}` key in `[python].resolves`.
+                """
+            )
+        else:
+            yield softwrap(
+                f"""
+                - The necessary requirements are specified by requirements targets marked with
+                `resolve="{resolve}"`.
+
+                - The lockfile destination is specified by the `{resolve}` key in
+                `[python].resolves`.
+                """
+            )
 
         if isinstance(metadata, PythonLockfileMetadataV2):
             # Note that by the time we have gotten to this error message, we should have already
             # validated that the transitive closure is using the same resolve, via
             # pex_from_targets.py. This implies that we don't need to worry about users depending
             # on python_requirement targets that aren't in that code's resolve.
             not_in_lock = sorted(str(r) for r in user_requirements - metadata.requirements)
-            yield f"- The requirements not provided by the `{resolve}` resolve are: "
+            yield f"\n\n- The requirements not provided by the `{resolve}` resolve are:\n  "
             yield str(not_in_lock)
-            yield "\n\n"
 
     if InvalidPythonLockfileReason.INTERPRETER_CONSTRAINTS_MISMATCH in validation.failure_reasons:
+        yield "\n\n"
         yield softwrap(
             f"""
             - The inputs use interpreter constraints (`{user_interpreter_constraints}`) that
             are not a subset of those used to generate the lockfile
             (`{metadata.valid_for_interpreter_constraints}`).
-
-            If `{resolve}` is a Python tool, the input interpreter constraints may be
-            specified by `[{resolve}].interpreter_constraints` (if applicable).
-
-            Otherwise, the input interpreter constraints are specified by your code, using
-            the `[python].interpreter_constraints` option and the `interpreter_constraints`
-            target field.
-
-            To create a lockfile with new interpreter constraints, update the option
-            `[python].resolves_to_interpreter_constraints`, and then generate the lockfile
-            (see below).
-
-            See {doc_url('python-interpreter-compatibility')} for details.
             """
-        ) + "\n\n"
+        )
+        if is_old_style_tool_lockfile:
+            yield softwrap(
+                """
+                - The input interpreter constraints may be specified by
+                `[{resolve}].interpreter_constraints` (if applicable).
+                """
+            )
+        else:
+            yield softwrap(
+                """
+                - The input interpreter constraints are specified by your code, using
+                the `[python].interpreter_constraints` option and the `interpreter_constraints`
+                target field.
+
+                - To create a lockfile with new interpreter constraints, update the option
+                `[python].resolves_to_interpreter_constraints`, and then generate the lockfile
+                (see below).
+                """
+            )
+        yield "\n\nSee {doc_url('python-interpreter-compatibility')} for details."
 
+    yield "\n\n"
     yield from _common_failure_reasons(validation.failure_reasons, maybe_constraints_file_path)
-
     yield "To regenerate your lockfile, "
     yield f"run `{bin_name()} generate-lockfiles --resolve={resolve}`."
+    yield f"\n\nSee {doc_url('python-third-party-dependencies')} for details.\n\n"
 
 
 def rules():

src/python/pants/backend/python/util_rules/pex_requirements_test.py

@@ -49,6 +49,8 @@ def create_python_setup(
         invalid_lockfile_behavior=behavior,
         resolves_generate_lockfiles=enable_resolves,
         interpreter_versions_universe=PythonSetup.default_interpreter_universe,
+        resolves={"a": "lock.txt"},
+        default_resolve="a",
     )
 
 
@@ -189,7 +191,7 @@ def contains(msg: str, if_: bool = True) -> None:
         if_=invalid_reqs,
     )
     contains(
-        "The requirements not provided by the `a` resolve are: ['bad-req']",
+        "The requirements not provided by the `a` resolve are:\n  ['bad-req']",
         if_=invalid_reqs,
     )
 

src/python/pants/core/goals/update_build_files_test.py

@@ -153,6 +153,7 @@ def assert_ics(
             lockfile=lckfile,
             interpreter_constraints=ics,
             version="v",
+            install_from_resolve=None,
             extra_requirements=[],
         )
         loaded_lock = LoadedLockfile(