Refactor V2 PEX creation rules (pantsbuild#8080)

pantsbuild#8063 was originally going to result in creating a `SourcesPex` (in addition to `RequirementsPex`). In preparation for that, the below refactors were made. Even though the approach to pantsbuild#8063 has changed, these are good changes to make in general and may be useful to porting `./pants binary`.

* Extract out `download_pex_bin` rule.
* Rename `resolve_requirements.py` -> `create_requirements_pex.py` for more explicit name (and avoid name clash with V1 file).
* Fix shadowing variable name.
* Use long CLI args.

src/python/pants/backend/python/register.py

@@ -5,7 +5,8 @@
 from pants.backend.python.python_artifact import PythonArtifact
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.python_requirements import PythonRequirements
-from pants.backend.python.rules import inject_init, python_test_runner, resolve_requirements
+from pants.backend.python.rules import (create_requirements_pex, download_pex_bin, inject_init,
+                                        python_test_runner)
 from pants.backend.python.subsystems.python_native_code import PythonNativeCode
 from pants.backend.python.subsystems.python_native_code import rules as python_native_code_rules
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEnvironment
@@ -88,9 +89,10 @@ def register_goals():
 
 def rules():
   return (
+    download_pex_bin.rules() +
     inject_init.rules() +
     python_test_runner.rules() +
     python_native_code_rules() +
-    resolve_requirements.rules() +
+    create_requirements_pex.rules() +
     subprocess_environment_rules()
   )

src/python/pants/backend/python/rules/resolve_requirements.py → src/python/pants/backend/python/rules/create_requirements_pex.py

@@ -1,17 +1,18 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from pants.backend.python.rules.download_pex_bin import DownloadedPexBin
 from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment, PythonNativeCode
 from pants.backend.python.subsystems.python_setup import PythonSetup
-from pants.engine.fs import Digest, Snapshot, UrlToFetch
+from pants.engine.fs import Digest
 from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult
 from pants.engine.rules import optionable_rule, rule
 from pants.engine.selectors import Get
 from pants.util.objects import datatype, hashable_string_list, string_optional, string_type
 from pants.util.strutil import create_path_env_var
 
 
-class ResolveRequirementsRequest(datatype([
+class RequirementsPexRequest(datatype([
   ('output_filename', string_type),
   ('requirements', hashable_string_list),
   ('interpreter_constraints', hashable_string_list),
@@ -20,22 +21,17 @@ class ResolveRequirementsRequest(datatype([
   pass
 
 
-class ResolvedRequirementsPex(datatype([('directory_digest', Digest)])):
+class RequirementsPex(datatype([('directory_digest', Digest)])):
   pass
 
 
 # TODO: This is non-hermetic because the requirements will be resolved on the fly by
 # pex, where it should be hermetically provided in some way.
-@rule(ResolvedRequirementsPex, [ResolveRequirementsRequest, PythonSetup, PexBuildEnvironment])
-def resolve_requirements(request, python_setup, pex_build_environment):
+@rule(RequirementsPex, [RequirementsPexRequest, DownloadedPexBin, PythonSetup, PexBuildEnvironment])
+def create_requirements_pex(request, pex_bin, python_setup, pex_build_environment):
   """Returns a PEX with the given requirements, optional entry point, and optional
   interpreter constraints."""
 
-  # TODO: Inject versions and digests here through some option, rather than hard-coding it.
-  url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex'
-  digest = Digest('2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441)
-  pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
-
   interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths)
   env = {"PATH": interpreter_search_paths, **pex_build_environment.invocation_environment_dict}
 
@@ -50,28 +46,26 @@ def resolve_requirements(request, python_setup, pex_build_environment):
   # necessarily the interpreter that PEX will use to execute the generated .pex file.
   # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
   # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
-  argv = ["python", "./{}".format(pex_snapshot.files[0]), "-o", request.output_filename]
+  argv = ["python", f"./{pex_bin.executable}", "--output-file", request.output_filename]
   if request.entry_point is not None:
-    argv.extend(["-e", request.entry_point])
+    argv.extend(["--entry-point", request.entry_point])
   argv.extend(interpreter_constraint_args + list(request.requirements))
 
-  request = ExecuteProcessRequest(
+  execute_process_request = ExecuteProcessRequest(
     argv=tuple(argv),
     env=env,
-    input_files=pex_snapshot.directory_digest,
-    description='Resolve requirements: {}'.format(", ".join(request.requirements)),
+    input_files=pex_bin.directory_digest,
+    description=f"Create a requirements PEX: {', '.join(request.requirements)}",
     output_files=(request.output_filename,),
   )
 
-  result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request)
-  yield ResolvedRequirementsPex(
-    directory_digest=result.output_directory_digest,
-  )
+  result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, execute_process_request)
+  yield RequirementsPex(directory_digest=result.output_directory_digest)
 
 
 def rules():
   return [
-    resolve_requirements,
+    create_requirements_pex,
     optionable_rule(PythonSetup),
     optionable_rule(PythonNativeCode),
   ]

src/python/pants/backend/python/rules/download_pex_bin.py

@@ -0,0 +1,26 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.engine.fs import Digest, Snapshot, UrlToFetch
+from pants.engine.rules import rule
+from pants.engine.selectors import Get
+from pants.util.objects import datatype
+
+
+class DownloadedPexBin(datatype([('executable', str), ('directory_digest', Digest)])):
+  pass
+
+
+@rule(DownloadedPexBin, [])
+def download_pex_bin():
+  # TODO: Inject versions and digests here through some option, rather than hard-coding it.
+  url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex'
+  digest = Digest('2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441)
+  snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
+  yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
+
+
+def rules():
+  return [
+    download_pex_bin,
+  ]

src/python/pants/backend/python/rules/python_test_runner.py

@@ -1,9 +1,9 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from pants.backend.python.rules.create_requirements_pex import (RequirementsPex,
+                                                                RequirementsPexRequest)
 from pants.backend.python.rules.inject_init import InjectedInitDigest
-from pants.backend.python.rules.resolve_requirements import (ResolvedRequirementsPex,
-                                                             ResolveRequirementsRequest)
 from pants.backend.python.subsystems.pytest import PyTest
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
@@ -53,7 +53,7 @@ def run_python_test(test_target, pytest, python_setup, source_root_config, subpr
         all_target_requirements.append(str(py_req.requirement))
   all_requirements = all_target_requirements + list(pytest.get_requirement_strings())
   resolved_requirements_pex = yield Get(
-    ResolvedRequirementsPex, ResolveRequirementsRequest(
+    RequirementsPex, RequirementsPexRequest(
       output_filename=output_pytest_requirements_pex_filename,
       requirements=tuple(sorted(all_requirements)),
       interpreter_constraints=tuple(sorted(interpreter_constraints)),

tests/python/pants_test/backend/python/rules/BUILD

@@ -1,6 +1,22 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+python_tests(
+  name='create_requirements_pex',
+  source='test_create_requirements_pex.py',
+  dependencies=[
+    'src/python/pants/backend/python/rules',
+    'src/python/pants/backend/python/subsystems',
+    'src/python/pants/engine:fs',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
+    'src/python/pants/util:collections',
+    'src/python/pants/util:contextutil',
+    'tests/python/pants_test:test_base',
+    'tests/python/pants_test/subsystem:subsystem_utils',
+  ]
+)
+
 python_tests(
   name='inject_init',
   source='test_inject_init.py',
@@ -22,18 +38,3 @@ python_tests(
   tags={'integration'},
 )
 
-python_tests(
-  name='resolve_requirements',
-  source='test_resolve_requirements.py',
-  dependencies=[
-    'src/python/pants/backend/python/rules',
-    'src/python/pants/backend/python/subsystems',
-    'src/python/pants/engine:fs',
-    'src/python/pants/engine:rules',
-    'src/python/pants/engine:selectors',
-    'src/python/pants/util:collections',
-    'src/python/pants/util:contextutil',
-    'tests/python/pants_test:test_base',
-    'tests/python/pants_test/subsystem:subsystem_utils',
-  ]
-)

tests/python/pants_test/backend/python/rules/test_resolve_requirements.py → tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py

@@ -5,9 +5,10 @@
 import os.path
 import zipfile
 
-from pants.backend.python.rules.resolve_requirements import (ResolvedRequirementsPex,
-                                                             ResolveRequirementsRequest,
-                                                             resolve_requirements)
+from pants.backend.python.rules.create_requirements_pex import (RequirementsPex,
+                                                                RequirementsPexRequest,
+                                                                create_requirements_pex)
+from pants.backend.python.rules.download_pex_bin import download_pex_bin
 from pants.backend.python.subsystems.python_native_code import (PythonNativeCode,
                                                                 create_pex_native_build_environment)
 from pants.backend.python.subsystems.python_setup import PythonSetup
@@ -25,31 +26,32 @@ class TestResolveRequirements(TestBase):
   @classmethod
   def rules(cls):
     return super().rules() + [
-      resolve_requirements,
-      RootRule(ResolveRequirementsRequest),
+      create_requirements_pex,
+      create_pex_native_build_environment,
+      download_pex_bin,
+      RootRule(RequirementsPexRequest),
       RootRule(PythonSetup),
       RootRule(PythonNativeCode),
-      create_pex_native_build_environment,
     ]
 
   def setUp(self):
     super().setUp()
     init_subsystems([PythonSetup, PythonNativeCode])
 
   def create_pex_and_get_pex_info(
-    self, requirements=None, entry_point=None, interpreter_constraints=None
+    self, *, requirements=None, entry_point=None, interpreter_constraints=None
   ):
     def hashify_optional_collection(iterable):
       return tuple(sorted(iterable)) if iterable is not None else tuple()
 
-    request = ResolveRequirementsRequest(
+    request = RequirementsPexRequest(
       output_filename="test.pex",
       requirements=hashify_optional_collection(requirements),
       interpreter_constraints=hashify_optional_collection(interpreter_constraints),
       entry_point=entry_point,
     )
     requirements_pex = assert_single_element(
-      self.scheduler.product_request(ResolvedRequirementsPex, [Params(
+      self.scheduler.product_request(RequirementsPex, [Params(
         request,
         PythonSetup.global_instance(),
         PythonNativeCode.global_instance()
@@ -64,19 +66,19 @@ def hashify_optional_collection(iterable):
           pex_info_content = pex_info.readline().decode()
     return json.loads(pex_info_content)
 
-  def test_resolves_dependencies(self):
+  def test_resolves_dependencies(self) -> None:
     requirements = {"six==1.12.0", "jsonschema==2.6.0", "requests==2.22.0"}
     pex_info = self.create_pex_and_get_pex_info(requirements=requirements)
     # NB: We do not check for transitive dependencies, which PEX-INFO will include. We only check
     # that at least the dependencies we requested are included.
     self.assertTrue(requirements.issubset(pex_info["requirements"]))
 
-  def test_entry_point(self):
+  def test_entry_point(self) -> None:
     entry_point = "pydoc"
     pex_info = self.create_pex_and_get_pex_info(entry_point=entry_point)
     self.assertEqual(pex_info["entry_point"], entry_point)
 
-  def test_interpreter_constraints(self):
+  def test_interpreter_constraints(self) -> None:
     constraints = {"CPython>=2.7,<3", "CPython>=3.6,<4"}
     pex_info = self.create_pex_and_get_pex_info(interpreter_constraints=constraints)
     self.assertEqual(set(pex_info["interpreter_constraints"]), constraints)