Revert "A python_binary_create task, to bring pex-building under goal."

This reverts commit eb94134.

BUILD

@@ -10,6 +10,6 @@ source_root('src/scala', jvm_binary, page, scala_library)
 source_root('src/thrift', java_thrift_library, page, python_thrift_library)
 
 source_root('tests/java', java_library, junit_tests, page)
-source_root('tests/python', page, python_library, python_tests, python_test_suite, python_binary, resources)
+source_root('tests/python', page, python_library, python_tests, python_test_suite, resources)
 source_root('tests/resources', page, resources)
 source_root('tests/scala', page, junit_tests, scala_library, scala_specs)

pants.ini

@@ -283,6 +283,8 @@ repos: [
 
 indices: ['pypi.python.org']
 
+allow_pypi: True
+
 
 [python-ipython]
 entry_point: IPython:start_ipython

src/python/pants/backend/python/register.py

@@ -3,7 +3,6 @@
 
 from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
                         print_function, unicode_literals)
-from pants.backend.python.tasks.python_binary_create import PythonBinaryCreate
 from pants.backend.python.tasks.python_run_tests import PythonRunTests
 
 from pants.goal import Goal as goal
@@ -65,9 +64,6 @@ def register_goals():
   goal(name='python-setup', action=SetupPythonEnvironment
   ).install('setup').with_description("Setup the target's build environment.")
 
-  goal(name='python-binary-create', action=PythonBinaryCreate, dependencies=['bootstrap', 'check-exclusives', 'resources']
-  ).install('binary')
-
   goal(name='pytest', action=PythonRunTests, dependencies=['bootstrap', 'check-exclusives', 'resources']
   ).install('test')
 

src/python/pants/backend/python/targets/python_binary.py

@@ -11,6 +11,7 @@
 from twitter.common.python.pex_info import PexInfo
 
 from pants.base.build_manual import manual
+from pants.base.target import Target
 from pants.base.exceptions import TargetDefinitionException
 from pants.backend.python.targets.python_target import PythonTarget
 
@@ -36,6 +37,7 @@ def __init__(self,
                repositories=None,         # pex option
                indices=None,              # pex option
                ignore_errors=False,       # pex option
+               allow_pypi=False,          # pex option
                platforms=(),
                **kwargs):
     """
@@ -53,7 +55,6 @@ def __init__(self,
       be written to disk.
     :param repositories: a list of repositories to query for dependencies.
     :param indices: a list of indices to use for packages.
-    :param ignore_errors: should we ignore inability to resolve dependencies?
     :param platforms: extra platforms to target when building this binary.
     :param compatibility: either a string or list of strings that represents
       interpreter compatibility for this target, using the Requirement-style format,

src/python/pants/backend/python/targets/python_target.py

@@ -44,9 +44,8 @@ def __init__(self,
 
     self._provides = provides
 
-    self._compatibility = maybe_list(compatibility or ())
-    # Check that the compatibility requirements are well-formed.
-    for req in self._compatibility:
+    self.compatibility = maybe_list(compatibility or ())
+    for req in self.compatibility:
       try:
         PythonIdentity.parse_requirement(req)
       except ValueError as e:
@@ -71,10 +70,6 @@ def provides(self):
         self._provides._binaries[key] = self._build_graph.get_target(address)
     return self._provides
 
-  @property
-  def compatibility(self):
-    return self._compatibility
-
   @property
   def resources(self):
     resource_targets = []

src/python/pants/backend/python/tasks/BUILD

@@ -4,7 +4,6 @@ python_library(
   dependencies = [
     pants('3rdparty/python/twitter/commons:twitter.common.collections'),
     pants('3rdparty/python/twitter/commons:twitter.common.contextutil'),
-    pants('3rdparty/python/twitter/commons:twitter.common.python'),
     pants('src/python/pants/base:exceptions'),
     pants('src/python/pants/base:target'),
     pants('src/python/pants/base:workunit'),

src/python/pants/backend/python/tasks/python_run_tests.py

@@ -20,13 +20,14 @@ class PythonRunTests(PythonTask):
   @classmethod
   def setup_parser(cls, option_group, args, mkflag):
     super(PythonRunTests, cls).setup_parser(option_group, args, mkflag)
+    # TODO(benjy): Support pass-thru of pytest flags.
     option_group.add_option(mkflag('fast'), mkflag('fast', negate=True),
                             dest='python_run_tests_fast',
                             action='callback', callback=mkflag.set_bool, default=True,
                             help='[%default] Run all tests in a single chroot. If set to false, '
                                  'each test target will create a new chroot, which will be much '
                                  'slower.')
-    # TODO(benjy): Support direct passthru of pytest flags.
+
     option_group.add_option(mkflag('options'),
                             dest='python_run_tests_options',
                             action='append', default=[],
@@ -46,14 +47,10 @@ def is_python_test(target):
 
       # TODO(benjy): Only color on terminals that support it.
       args = ['--color', 'yes']
-      # TODO(benjy): A less hacky way to find the log level.
-      if self.context.options.log_level == 'debug':
-        args.append('-s')  # Make pytest emit all stdout/stderr, even for successful tests.
       if self.context.options.python_run_tests_options:
         for options in self.context.options.python_run_tests_options:
           args.extend(shlex.split(options))
-      test_builder = PythonTestBuilder(targets=test_targets,
-                                       args=args,
+      test_builder = PythonTestBuilder(test_targets, args,
                                        interpreter=self.interpreter,
                                        conn_timeout=self.conn_timeout,
                                        fast=self.context.options.python_run_tests_fast)

src/python/pants/backend/python/tasks/python_task.py

@@ -23,28 +23,21 @@ def setup_parser(cls, option_group, args, mkflag):
 
   def __init__(self, context, workdir):
     super(PythonTask, self).__init__(context, workdir)
-    self.conn_timeout = (self.context.options.python_conn_timeout or
+    self.conn_timeout = (self.context.options.python_conn_timeout or 
                          self.context.config.getdefault('connection_timeout'))
 
     self.interpreter_cache = PythonInterpreterCache(self.context.config,
                                                     logger=self.context.log.debug)
-    self.interpreter_cache.setup()
-
-    # Select a default interpreter to use.
-    compatibilities = self.context.options.python_interpreters or [b'']
-    self._interpreter = self.select_interpreter(compatibilities)
+    interpreters = self.context.options.python_interpreters or [b'']
+    self.interpreter_cache.setup(filters=interpreters)
+    interpreters = self.interpreter_cache.select_interpreter(
+      list(self.interpreter_cache.matches(interpreters)))
+    if len(interpreters) != 1:
+      raise TaskError('Unable to detect suitable interpreter.')
+    else:
+      self.context.log.debug('Selected %s' % interpreters[0])
+    self._interpreter = interpreters[0]
 
   @property
   def interpreter(self):
-    """Subclasses can use this if they're fine with the default interpreter (the usual case)."""
     return self._interpreter
-
-  def select_interpreter(self, compatibilities):
-    """Subclasses can use this to be more specific about interpreter selection."""
-    interpreters = self.interpreter_cache.select_interpreter(
-      list(self.interpreter_cache.matches(compatibilities)))
-    if len(interpreters) != 1:
-      raise TaskError('Unable to detect suitable interpreter.')
-    interpreter = interpreters[0]
-    self.context.log.debug('Selected %s' % interpreter)
-    return interpreter

src/python/pants/engine/linear_engine.py

@@ -116,10 +116,7 @@ def attempt(self, context, phases):
     # Then we grab the lock at the beginning of f's execution and don't relinquish until the largest
     # scope serialization requirement from c is past.
     serialized_phase_executors = [pe for pe in phase_executors if pe.phase.serialize]
-    if context.options.no_lock:
-      outer_lock_holder = None
-    else:
-      outer_lock_holder = serialized_phase_executors[-1] if serialized_phase_executors else None
+    outer_lock_holder = serialized_phase_executors[-1] if serialized_phase_executors else None
 
     if outer_lock_holder:
       context.acquire_lock()

src/python/pants/goal/option_helpers.py

@@ -27,16 +27,13 @@ def _set_bool(option, opt_str, value, parser):
          help='Kill nailguns before exiting'),
   Option('-d', '--logdir', dest='logdir',
          help='[%default] Write logs to files under this directory.'),
-  Option('-l', '--level', dest='log_level', type='choice', choices=['debug', 'info', 'warn'],
-         default='info', help='[%default] Set the logging level [debug, info, warn].'),
+  Option('-l', '--level', dest='log_level', type='choice', choices=['debug', 'info', 'warn'], default='info',
+         help='[%default] Set the logging level [debug, info, warn].'),
   Option('-q', '--quiet', action='store_true', dest='quiet', default=False,
          help='Squelches all console output apart from errors.'),
   Option('--no-colors', dest='no_color', action='store_true', default=False,
          help='Do not colorize log messages.'),
-  Option('--no-lock', dest='no_lock', action='store_true', default=False,
-         help="Don't attempt to grab the global lock. This lock prevents two concurrent pants "
-              "instances from stomping on each others data, so only use this if you know what "
-              "you're doing."),
+
   Option('--read-from-artifact-cache', '--no-read-from-artifact-cache', action='callback',
          callback=_set_bool, dest='read_from_artifact_cache', default=True,
          help='Read build artifacts from cache, if available.'),

tests/python/pants_test/BUILD

@@ -54,7 +54,6 @@ python_test_suite(
   name = 'integration',
   dependencies = [
     pants('tests/python/pants_test/tasks:jar_publish_integration'),
-    pants('tests/python/pants_test/python:interpreter_selection_integration'),
   ]
 )
 

tests/python/pants_test/engine/test_linear_engine.py

@@ -15,7 +15,7 @@ class LinearEngineTest(EngineTestBase, BaseTest):
   def setUp(self):
     super(LinearEngineTest, self).setUp()
 
-    self._context = self.context(options=dict(explain=False, no_lock=False))
+    self._context = self.context(options=dict(explain=False))
     self.assertTrue(self._context.is_unlocked())
 
     self.engine = LinearEngine()

tests/python/pants_test/pants_run_integration_test.py

@@ -4,9 +4,11 @@
 from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
                         print_function, unicode_literals)
 
+
 from contextlib import contextmanager
 import os
 import subprocess
+from textwrap import dedent
 import unittest
 
 from twitter.common.contextutil import temporary_dir
@@ -16,42 +18,29 @@
 
 
 class PantsRunIntegrationTest(unittest.TestCase):
-  """A base class useful for integration tests for targets in the same repo."""
+  """A baseclass useful for integration tests for targets in the same repo"""
 
   PANTS_SUCCESS_CODE = 0
   PANTS_SCRIPT_NAME = 'pants'
 
   @contextmanager
-  def run_pants(self, command, config=None, **kwargs):
+  def run_pants(self, command, **kwargs):
     """Runs pants in a subprocess.
 
     :param list command: A list of command line arguments coming after `./pants`.
-    :param config: Optional data for a generated ini file. A map of <section-name> ->
-    map of key -> value. If order in the ini file matters, this should be an OrderedDict.
     :param kwargs: Extra keyword args to pass to `subprocess.Popen`.
-
-    IMPORTANT NOTE: The subprocess will be run with --no-lock, so that it doesn't deadlock waiting
-    for this process to release the workspace lock. It's the caller's responsibility to ensure
-    that the invoked pants doesn't interact badly with this one.
     """
-    config = config.copy() if config else {}
-    with temporary_dir() as workdir:
-      # We add workdir to the DEFAULT section, and also ensure that it's emitted first.
-      default_section = config.pop('DEFAULT', {})
-      default_section['pants_workdir'] = '%s' % workdir
-
-      ini = ''
-      for section, section_config in [('DEFAULT', default_section)] + config.items():
-        ini += '\n[%s]\n' % section
-        for key, val in section_config.items():
-          ini += '%s: %s\n' % (key, val)
-
-      ini_file_name = os.path.join(workdir, 'pants.ini')
+    with temporary_dir() as work_dir:
+      ini = dedent('''
+              [DEFAULT]
+              pants_workdir:  %(workdir)s
+              ''' % dict(workdir=work_dir))
+
+      ini_file_name = os.path.join(work_dir, 'pants.ini')
       with safe_open(ini_file_name, mode='w') as fp:
         fp.write(ini)
       env = os.environ.copy()
       env['PANTS_CONFIG_OVERRIDE'] = ini_file_name
-      pants_command = [os.path.join(get_buildroot(), self.PANTS_SCRIPT_NAME)] + command + ['--no-lock']
-      print('RUNNING %s' % ' '.join(pants_command))
+      pants_command = [os.path.join(get_buildroot(), self.PANTS_SCRIPT_NAME)] + command
       result = subprocess.call(pants_command, env=env, **kwargs)
       yield result