An option to test/run/repl against the entire lockfile. (pantsbuild#1…

…3732)

This cuts out the subsetting, which can add a lot of time
to runs, and can also bloat the cache dramatically.

One downside is less hermetic runs - you may be incidentally
relying on requirements that you don't depend on but happen to
be present in the lockfile (but this seems like less
of an issue with dep inference).

A more significant downside is that any changes to the lockfile
will invalidate all tests. Still, this is a performance knob that
users can decide how to use. In many cases that may be a very
acceptable consequence.

[ci skip-rust]

[ci skip-build-wheels]

src/python/pants/backend/python/subsystems/setup.py

@@ -149,6 +149,24 @@ def register_options(cls, register):
                 "We recommend keeping the default of `error` for CI builds."
             ),
         )
+        register(
+            "--run-against-entire-lockfile",
+            advanced=True,
+            default=False,
+            type=bool,
+            help=(
+                "If enabled, when running binaries, tests, and repls, Pants will use the entire "
+                "lockfile/constraints file instead of just the relevant subset. This can improve "
+                "performance and reduce cache size, but has two consequences: 1) All cached test "
+                "results will be invalidated if any requirement in the lockfile changes, rather "
+                "than just those that depend on the changed requirement. 2) Requirements unneeded "
+                "by a test/run/repl will be present on the sys.path, which might in rare cases "
+                "cause their behavior to change.\n\n"
+                "This option does not affect packaging deployable artifacts, such as "
+                "PEX files, wheels and cloud functions, which will still use just the exact "
+                "subset of requirements needed."
+            ),
+        )
         register(
             "--interpreter-search-paths",
             advanced=True,
@@ -261,6 +279,10 @@ def resolves_to_lockfiles(self) -> dict[str, str]:
     def invalid_lockfile_behavior(self) -> InvalidLockfileBehavior:
         return cast(InvalidLockfileBehavior, self.options.invalid_lockfile_behavior)
 
+    @property
+    def run_against_entire_lockfile(self) -> bool:
+        return cast(bool, self.options.run_against_entire_lockfile)
+
     @property
     def resolve_all_constraints(self) -> bool:
         return cast(bool, self.options.resolve_all_constraints)

src/python/pants/backend/python/util_rules/pex_from_targets.py

@@ -48,6 +48,7 @@
     TransitiveTargets,
     TransitiveTargetsRequest,
 )
+from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 from pants.util.meta import frozen_after_init
 from pants.util.ordered_set import FrozenOrderedSet
@@ -582,15 +583,34 @@ def __init__(
 
 
 @rule
-async def get_requirements_pex(request: RequirementsPexRequest) -> PexRequest:
+async def get_requirements_pex(request: RequirementsPexRequest, setup: PythonSetup) -> PexRequest:
+    if setup.run_against_entire_lockfile and request.internal_only:
+        opt_pex_request = await Get(
+            OptionalPexRequest,
+            _RepositoryPexRequest(
+                addresses=sorted(request.addresses),
+                internal_only=request.internal_only,
+                hardcoded_interpreter_constraints=request.hardcoded_interpreter_constraints,
+                direct_deps_only=request.direct_deps_only,
+                resolve_and_lockfile=request.resolve_and_lockfile,
+            ),
+        )
+        if opt_pex_request.maybe_pex_request is None:
+            raise ValueError(
+                "[python].run_against_entire_lockfile was set, but could not find a "
+                "lockfile or constraints file for this target set. See "
+                f"{doc_url('python-third-party-dependencies')} for details."
+            )
+        return opt_pex_request.maybe_pex_request
+
     pex_request = await Get(
         PexRequest,
         PexFromTargetsRequest(
             addresses=sorted(request.addresses),
             output_filename="requirements.pex",
+            internal_only=request.internal_only,
             include_source_files=False,
             hardcoded_interpreter_constraints=request.hardcoded_interpreter_constraints,
-            internal_only=request.internal_only,
             direct_deps_only=request.direct_deps_only,
             resolve_and_lockfile=request.resolve_and_lockfile,
         ),