Make PexBuilderWrapper a Subsystem. (pantsbuild#6897)

This allows it to be "injected" directly into consumers, eliminating
the middleman.

contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py

@@ -10,11 +10,10 @@
 from packaging import version
 from pants.backend.python.interpreter_cache import PythonInterpreterCache
 from pants.backend.python.python_requirement import PythonRequirement
-from pants.backend.python.subsystems.python_repos import PythonRepos
+from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
-from pants.backend.python.tasks.pex_build_util import PexBuilderWrapper
 from pants.base.build_environment import get_buildroot, pants_version
 from pants.base.deprecated import deprecated_conditional
 from pants.base.exceptions import TaskError
@@ -58,7 +57,9 @@ def plugin_subsystems(cls):
   @classmethod
   def subsystem_dependencies(cls):
     return super(Task, cls).subsystem_dependencies() + cls.plugin_subsystems + (
-      PythonSetup, PythonRepos, PythonInterpreterCache
+      PexBuilderWrapper.Factory,
+      PythonInterpreterCache,
+      PythonSetup,
     )
 
   @classmethod
@@ -123,10 +124,9 @@ def checker_pex(self, interpreter):
     if not os.path.exists(pex_path):
       with self.context.new_workunit(name='build-checker'):
         with safe_concurrent_creation(pex_path) as chroot:
-          pex_builder = PexBuilderWrapper(
-            PEXBuilder(path=chroot, interpreter=interpreter),
-            PythonRepos.global_instance(),
-            PythonSetup.global_instance(), self.context.log)
+          pex_builder = PexBuilderWrapper.Factory.create(
+            builder=PEXBuilder(path=chroot, interpreter=interpreter),
+            log=self.context.log)
 
           # Constraining is required to guard against the case where the user
           # has a pexrc file set.

contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py

@@ -10,13 +10,12 @@
 from builtins import open, str
 
 from pants.backend.python.interpreter_cache import PythonInterpreterCache
-from pants.backend.python.subsystems.python_repos import PythonRepos
-from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.pex_build_util import (PexBuilderWrapper,
+                                                            has_python_requirements,
+                                                            has_python_sources)
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.targets.python_target import PythonTarget
-from pants.backend.python.tasks.pex_build_util import (PexBuilderWrapper, has_python_requirements,
-                                                       has_python_sources)
 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase
 from pants.base.exceptions import TaskError
 from pants.base.generator import Generator, TemplateData
@@ -46,7 +45,8 @@ def __init__(self, *args, **kwargs):
   @classmethod
   def subsystem_dependencies(cls):
     return super(PythonEval, cls).subsystem_dependencies() + (
-      PythonRepos, PythonSetup, PythonInterpreterCache
+      PexBuilderWrapper.Factory,
+      PythonInterpreterCache
     )
 
   @classmethod
@@ -217,11 +217,9 @@ def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt):
     if not os.path.isdir(reqs_pex_path):
       req_libs = [t for t in vt.target.closure() if has_python_requirements(t)]
       with safe_concurrent_creation(reqs_pex_path) as safe_path:
-        pex_builder = PexBuilderWrapper(
-          PEXBuilder(safe_path, interpreter=interpreter, copy=True),
-          PythonRepos.global_instance(),
-          PythonSetup.global_instance(),
-          self.context.log)
+        pex_builder = PexBuilderWrapper.Factory.create(
+          builder=PEXBuilder(safe_path, interpreter=interpreter, copy=True),
+          log=self.context.log)
         pex_builder.add_requirement_libs_from(req_libs)
         pex_builder.freeze()
     return PEX(reqs_pex_path, interpreter=interpreter)
@@ -234,11 +232,9 @@ def _source_pex_for_versioned_target_closure(self, interpreter, vt):
     return PEX(source_pex_path, interpreter=interpreter)
 
   def _build_source_pex(self, interpreter, path, targets):
-    pex_builder = PexBuilderWrapper(
-      PEXBuilder(path=path, interpreter=interpreter, copy=True),
-      PythonRepos.global_instance(),
-      PythonSetup.global_instance(),
-      self.context.log)
+    pex_builder = PexBuilderWrapper.Factory.create(
+      builder=PEXBuilder(path=path, interpreter=interpreter, copy=True),
+      log=self.context.log)
     for target in targets:
       if has_python_sources(target):
         pex_builder.add_sources_from(target)

src/python/pants/backend/project_info/tasks/export.py

@@ -23,10 +23,10 @@
 from pants.backend.jvm.tasks.coursier_resolve import CoursierMixin
 from pants.backend.jvm.tasks.ivy_task_mixin import IvyTaskMixin
 from pants.backend.python.interpreter_cache import PythonInterpreterCache
+from pants.backend.python.subsystems.pex_build_util import has_python_requirements
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.backend.python.targets.python_tests import PythonTests
-from pants.backend.python.tasks.pex_build_util import has_python_requirements
 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError

src/python/pants/backend/python/subsystems/BUILD

@@ -6,6 +6,11 @@ python_library(
     '3rdparty/python:future',
     '3rdparty/python:pex',
     '3rdparty/python:setuptools',
+    '3rdparty/python/twitter/commons:twitter.common.collections',
+    'src/python/pants/base:build_environment',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/backend/python/targets',
+    'src/python/pants/build_graph',
     'src/python/pants/option',
     'src/python/pants/subsystem',
     'src/python/pants/util:memo',

src/python/pants/backend/python/tasks/pex_build_util.py → src/python/pants/backend/python/subsystems/pex_build_util.py

@@ -4,14 +4,18 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import logging
 import os
 from builtins import str
 
 from future.utils import PY2
 from pex.fetcher import Fetcher
+from pex.pex_builder import PEXBuilder
 from pex.resolver import resolve
 from twitter.common.collections import OrderedSet
 
+from pants.backend.python.subsystems.python_repos import PythonRepos
+from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_library import PythonLibrary
@@ -20,6 +24,7 @@
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.build_graph.files import Files
+from pants.subsystem.subsystem import Subsystem
 
 
 def is_python_target(tgt):
@@ -87,7 +92,30 @@ def dump_sources(builder, tgt, log):
 class PexBuilderWrapper(object):
   """Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities."""
 
-  def __init__(self, builder, python_repos_subsystem, python_setup_subsystem, log=None):
+  class Factory(Subsystem):
+    options_scope = 'pex-builder-wrapper'
+
+    @classmethod
+    def subsystem_dependencies(cls):
+      return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + (
+        PythonRepos,
+        PythonSetup,
+      )
+
+    @classmethod
+    def create(cls, builder, log=None):
+      log = log or logging.getLogger(__name__)
+      return PexBuilderWrapper(builder=builder,
+                               python_repos_subsystem=PythonRepos.global_instance(),
+                               python_setup_subsystem=PythonSetup.global_instance(),
+                               log=log)
+
+  def __init__(self, builder, python_repos_subsystem, python_setup_subsystem, log):
+    assert isinstance(builder, PEXBuilder)
+    assert isinstance(python_repos_subsystem, PythonRepos)
+    assert isinstance(python_setup_subsystem, PythonSetup)
+    assert log is not None
+
     self._builder = builder
     self._python_repos_subsystem = python_repos_subsystem
     self._python_setup_subsystem = python_setup_subsystem

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -142,6 +142,14 @@ def check_build_for_current_platform_only(self, targets):
 class BuildSetupRequiresPex(ExecutablePexTool):
   options_scope = 'build-setup-requires-pex'
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(BuildSetupRequiresPex, cls).subsystem_dependencies() + (PythonSetup,)
+
+  @memoized_property
+  def python_setup(self):
+    return PythonSetup.global_instance()
+
   @property
   def base_requirements(self):
     # TODO: would we ever want to configure these requirement versions separately from the global

src/python/pants/backend/python/targets/BUILD

@@ -14,19 +14,13 @@ python_library(
   dependencies = [
     '3rdparty/python:future',
     '3rdparty/python:pex',
-    '3rdparty/python:setuptools',
-    '3rdparty/python:six',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/backend/python:python_artifact',
     'src/python/pants/backend/python:python_requirement',
-    'src/python/pants/backend/python/subsystems',
-    'src/python/pants/base:deprecated',
     'src/python/pants/base:exceptions',
     'src/python/pants/base:payload',
     'src/python/pants/base:payload_field',
     'src/python/pants/build_graph',
-    'src/python/pants/subsystem',
-    'src/python/pants/util:memo',
   ],
 )
 

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -18,12 +18,12 @@
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.native.tasks.link_shared_libraries import SharedLibrary
 from pants.backend.python.python_requirement import PythonRequirement
+from pants.backend.python.subsystems.pex_build_util import is_local_python_dist
 from pants.backend.python.subsystems.python_native_code import (BuildSetupRequiresPex,
                                                                 PythonNativeCode,
                                                                 SetupPyExecutionEnvironment,
                                                                 SetupPyNativeTools)
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
-from pants.backend.python.tasks.pex_build_util import is_local_python_dist
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.base.workunit import WorkUnitLabel

src/python/pants/backend/python/tasks/gather_sources.py

@@ -11,8 +11,8 @@
 from pex.pex_builder import PEXBuilder
 from twitter.common.collections import OrderedSet
 
-from pants.backend.python.tasks.pex_build_util import (PexBuilderWrapper, has_python_sources,
-                                                       has_resources, is_python_target)
+from pants.backend.python.subsystems.pex_build_util import (PexBuilderWrapper, has_python_sources,
+                                                            has_resources, is_python_target)
 from pants.base.exceptions import TaskError
 from pants.invalidation.cache_manager import VersionedTargetSet
 from pants.task.task import Task
@@ -33,6 +33,10 @@ class GatherSources(Task):
   def implementation_version(cls):
     return super(GatherSources, cls).implementation_version() + [('GatherSources', 5)]
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(GatherSources, cls).subsystem_dependencies() + (PexBuilderWrapper.Factory,)
+
   @classmethod
   def product_types(cls):
     return [cls.PYTHON_SOURCES]
@@ -82,10 +86,8 @@ def _get_pex_for_versioned_targets(self, interpreter, versioned_targets):
     return PEX(source_pex_path, interpreter=interpreter)
 
   def _build_pex(self, interpreter, path, targets):
-    pex_builder = PexBuilderWrapper(
-      PEXBuilder(path=path, interpreter=interpreter, copy=True),
-      python_repos_subsystem=None,
-      python_setup_subsystem=None,
+    pex_builder = PexBuilderWrapper.Factory.create(
+      builder=PEXBuilder(path=path, interpreter=interpreter, copy=True),
       log=self.context.log)
 
     for target in targets:

src/python/pants/backend/python/tasks/local_python_distribution_artifact.py

@@ -6,7 +6,7 @@
 
 import os
 
-from pants.backend.python.tasks.pex_build_util import is_local_python_dist
+from pants.backend.python.subsystems.pex_build_util import is_local_python_dist
 from pants.base.build_environment import get_buildroot
 from pants.task.task import Task
 from pants.util.dirutil import safe_mkdir

src/python/pants/backend/python/tasks/python_binary_create.py

@@ -10,14 +10,13 @@
 from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
 
+from pants.backend.python.subsystems.pex_build_util import (PexBuilderWrapper,
+                                                            has_python_requirements,
+                                                            has_python_sources, has_resources,
+                                                            is_python_target)
 from pants.backend.python.subsystems.python_native_code import PythonNativeCode
-from pants.backend.python.subsystems.python_repos import PythonRepos
-from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
-from pants.backend.python.tasks.pex_build_util import (PexBuilderWrapper, has_python_requirements,
-                                                       has_python_sources, has_resources,
-                                                       is_python_target)
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.build_graph.target_scopes import Scopes
@@ -34,9 +33,8 @@ class PythonBinaryCreate(Task):
   @classmethod
   def subsystem_dependencies(cls):
     return super(PythonBinaryCreate, cls).subsystem_dependencies() + (
+      PexBuilderWrapper.Factory,
       PythonNativeCode.scoped(cls),
-      PythonRepos,
-      PythonSetup,
     )
 
   @memoized_property
@@ -119,11 +117,9 @@ def _create_binary(self, binary_tgt, results_dir):
       pex_info = binary_tgt.pexinfo.copy()
       pex_info.build_properties = build_properties
 
-      pex_builder = PexBuilderWrapper(
-        PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True),
-        PythonRepos.global_instance(),
-        PythonSetup.global_instance(),
-        self.context.log)
+      pex_builder = PexBuilderWrapper.Factory.create(
+        builder=PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True),
+        log=self.context.log)
 
       if binary_tgt.shebang:
         self.context.log.info('Found Python binary target {} with customized shebang, using it: {}'

src/python/pants/backend/python/tasks/python_execution_task_base.py

@@ -11,12 +11,12 @@
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
 
+from pants.backend.python.subsystems.pex_build_util import is_python_target
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.backend.python.tasks.gather_sources import GatherSources
-from pants.backend.python.tasks.pex_build_util import is_python_target
 from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase
 from pants.build_graph.files import Files

src/python/pants/backend/python/tasks/python_tool_prep_base.py

@@ -11,10 +11,8 @@
 from pex.pex_builder import PEXBuilder
 
 from pants.backend.python.python_requirement import PythonRequirement
-from pants.backend.python.subsystems.python_repos import PythonRepos
-from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
-from pants.backend.python.tasks.pex_build_util import PexBuilderWrapper
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnitLabel
@@ -55,8 +53,7 @@ class PythonToolPrepBase(Task):
   def subsystem_dependencies(cls):
     return super(PythonToolPrepBase, cls).subsystem_dependencies() + (
       cls.tool_subsystem_cls.scoped(cls),
-      PythonSetup,
-      PythonRepos,
+      PexBuilderWrapper.Factory,
     )
 
   @classmethod
@@ -83,11 +80,9 @@ def _create_requirements(self, context, workdir):
 
   def _build_tool_pex(self, context, interpreter, pex_path, requirements_lib):
     with safe_concurrent_creation(pex_path) as chroot:
-      pex_builder = PexBuilderWrapper(
-        PEXBuilder(path=chroot, interpreter=interpreter),
-        PythonRepos.global_instance(),
-        PythonSetup.global_instance(),
-        context.log)
+      pex_builder = PexBuilderWrapper.Factory.create(
+        builder=PEXBuilder(path=chroot, interpreter=interpreter),
+        log=context.log)
       pex_builder.add_requirement_libs_from(req_libs=[requirements_lib])
       pex_builder.set_entry_point(self._tool_subsystem().get_entry_point())
       pex_builder.freeze()

src/python/pants/backend/python/tasks/resolve_requirements.py

@@ -6,7 +6,7 @@
 
 from pex.interpreter import PythonInterpreter
 
-from pants.backend.python.tasks.pex_build_util import has_python_requirements, is_python_target
+from pants.backend.python.subsystems.pex_build_util import has_python_requirements, is_python_target
 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase