Added Python setup to export goal to consume in the Pants Plugin for …

…IntelliJ.

Testing Done:
travis CI is passing + see https://rbcommons.com/s/twitter/r/2785/ for details on testing from the Plugin's side.

Bugs closed: 2157

Reviewed at https://rbcommons.com/s/twitter/r/2786/

build-support/pants-intellij.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# A script file to generate IntelliJ project from using the Intellij Pants Plugin.
+# Note: for any modification in this file please modify ExportIntegrationTest#test_intellij_integration
+
+# We don't want to include targets which are used in unit tests in our project so let's exclude them.
+./pants export src/python/:: tests/python/pants_test:all contrib/:: \
+    --exclude-target-regexp='.*go/examples.*' \
+    --exclude-target-regexp='.*scrooge/tests/thrift.*' \
+    --exclude-target-regexp='.*spindle/tests/thrift.*' \
+    --exclude-target-regexp='.*spindle/tests/jvm.*'

contrib/spindle/BUILD

@@ -3,5 +3,5 @@
 
 source_root('3rdparty', jar_library)
 source_root('src/python', page, python_binary, python_library, resources)
-source_root('tests/jvm', page, scala_library)
+source_root('tests/jvm', page, scala_library, junit_tests)
 source_root('tests/thrift', page, spindle_thrift_library)

src/python/pants/backend/project_info/tasks/BUILD

@@ -69,12 +69,14 @@ python_library(
   sources = ['export.py'],
   dependencies = [
     ':projectutils',
+    '3rdparty/python:pex',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/backend/core/targets:all',
     'src/python/pants/backend/core/tasks:console_task',
     'src/python/pants/backend/jvm/targets:jvm',
     'src/python/pants/backend/jvm/targets:scala',
     'src/python/pants/backend/jvm:ivy_utils',
+    'src/python/pants/backend/python/tasks:python',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:exceptions',
   ],

src/python/pants/backend/project_info/tasks/export.py

@@ -9,6 +9,7 @@
 import os
 from collections import defaultdict
 
+from pex.pex_info import PexInfo
 from twitter.common.collections import OrderedSet
 
 from pants.backend.core.targets.resources import Resources
@@ -20,14 +21,17 @@
 from pants.backend.jvm.targets.jvm_target import JvmTarget
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.backend.project_info.tasks.projectutils import get_jar_infos
+from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
+from pants.backend.python.targets.python_target import PythonTarget
+from pants.backend.python.tasks.python_task import PythonTask
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.java.distribution.distribution import DistributionLocator
 
 
 # Changing the behavior of this task may affect the IntelliJ Pants plugin
 # Please add fkorotkov, tdesai to reviews for this file
-class Export(ConsoleTask):
+class Export(PythonTask, ConsoleTask):
   """Generates a JSON description of the targets as configured in pants.
 
   Intended for exporting project information for IDE, such as the IntelliJ Pants plugin.
@@ -44,7 +48,7 @@ class Export(ConsoleTask):
   #
   # Note format changes in src/python/pants/docs/export.md and update the Changelog section.
   #
-  DEFAULT_EXPORT_VERSION='1.0.3'
+  DEFAULT_EXPORT_VERSION='1.0.4'
 
   @classmethod
   def subsystem_dependencies(cls):
@@ -130,6 +134,7 @@ def console_output(self, targets):
         ivy_info = ivy_info_list[0]
 
     ivy_jar_memo = {}
+    python_interpreter_targets_mapping = defaultdict(list)
 
     def process_target(current_target):
       """
@@ -175,6 +180,18 @@ def get_transitive_jars(jar_lib):
         if self.get_options().sources:
           info['sources'] = list(current_target.sources_relative_to_buildroot())
 
+      if isinstance(current_target, PythonRequirementLibrary):
+        reqs = current_target.payload.get_field_value('requirements', set())
+        """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
+        info['requirements'] = [req.key for req in reqs]
+
+      if isinstance(current_target, PythonTarget):
+        interpreter_for_target = self.select_interpreter_for_targets([current_target])
+        if interpreter_for_target is None:
+          raise TaskError('Unable to find suitable interpreter for {}'.format(current_target.address))
+        python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
+        info['python_interpreter'] = str(interpreter_for_target.identity)
+
       target_libraries = OrderedSet()
       if isinstance(current_target, JarLibrary):
         target_libraries = get_transitive_jars(current_target)
@@ -220,8 +237,9 @@ def get_transitive_jars(jar_lib):
     }
 
     graph_info = {
+      'version': self.DEFAULT_EXPORT_VERSION,
       'targets': targets_map,
-      'jvm_platforms' : jvm_platforms_map,
+      'jvm_platforms': jvm_platforms_map,
     }
     jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
     if jvm_distributions:
@@ -230,7 +248,26 @@ def get_transitive_jars(jar_lib):
     if self.get_options().libraries:
       graph_info['libraries'] = self._resolve_jars_info()
 
-    graph_info['version'] = self.DEFAULT_EXPORT_VERSION
+    if python_interpreter_targets_mapping:
+      default_interpreter = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys())[0]
+
+      interpreters_info = {}
+      for interpreter, targets in python_interpreter_targets_mapping.iteritems():
+        chroot = self.cached_chroot(
+          interpreter=interpreter,
+          pex_info=PexInfo.default(),
+          targets=targets
+        )
+        interpreters_info[str(interpreter.identity)] = {
+          'binary': interpreter.binary,
+          'chroot': chroot.path()
+        }
+
+      graph_info['python_setup'] = {
+        'default_interpreter': str(default_interpreter.identity),
+        'interpreters': interpreters_info
+      }
+
 
     if self.format:
       return json.dumps(graph_info, indent=4, separators=(',', ': ')).splitlines()

src/python/pants/backend/python/tasks/python_task.py

@@ -126,12 +126,14 @@ def create_chroot(self, interpreter, builder, targets, platforms, extra_requirem
                         extra_requirements=extra_requirements,
                         log=self.context.log)
 
-  def cached_chroot(self, interpreter, pex_info, targets, platforms,
+  def cached_chroot(self, interpreter, pex_info, targets, platforms=None,
                     extra_requirements=None, executable_file_content=None):
     """Returns a cached PythonChroot created with the specified args.
 
     The returned chroot will be cached for future use.
 
+    :rtype: pants.backend.python.python_chroot.PythonChroot
+
     TODO: Garbage-collect old chroots, so they don't pile up?
     TODO: Ideally chroots would just be products produced by some other task. But that's
           a bit too complicated to implement right now, as we'd need a way to request

src/python/pants/docs/export.md

@@ -16,6 +16,9 @@ See the Changelog section below for differences between versions of the export f
   - "libraries" : Dictionary of paths to .jar files resolved by pants required to compile the
   project.
   - "targets" : Dictionary of targets defined in BUILD files.
+  - "distributions" : Information about JVM distributions.
+  - "jvm_platforms" : Information about JVM platforms(language level and additionals arguments).
+  - "python_setup" : Information about Python setup(interpreters and chroots for them).
 
 ## Version
 
@@ -57,23 +60,20 @@ fields defined in the target:
 
 # Example export output
 
-The following is an abbreviated export file from the command in the pants repo:
-
-`./pants export ./examples/tests/java/org/pantsbuild/example/usethrift`
-
+The following is an abbreviated export file from a command in the pants repo:
 
 ```
 {
-    "version": "1.0.3",
+    "version": "1.0.4",
     "libraries": {
         "commons-logging:commons-logging:1.1.1": {
-            "default": "/Users/zundel/.ivy2/pants/commons-logging/commons-logging/jars/commons-logging-1.1.1.jar"
+            "default": "/Users/user/.ivy2/pants/commons-logging/commons-logging/jars/commons-logging-1.1.1.jar"
         },
         "commons-codec:commons-codec:1.6": {
-            "default": "/Users/zundel/.ivy2/pants/commons-codec/commons-codec/jars/commons-codec-1.6.jar"
+            "default": "/Users/user/.ivy2/pants/commons-codec/commons-codec/jars/commons-codec-1.6.jar"
         },
         "org.apache.httpcomponents:httpclient:4.2.5": {
-            "default": "/Users/zundel/.ivy2/pants/org.apache.httpcomponents/httpclient/jars/httpclient-4.2.5.jar"
+            "default": "/Users/user/.ivy2/pants/org.apache.httpcomponents/httpclient/jars/httpclient-4.2.5.jar"
         },
         ...
     },
@@ -98,6 +98,15 @@ The following is an abbreviated export file from the command in the pants repo:
         },
         "default_platform": "java6"
     },
+    "python_setup": {
+        "interpreters": {
+            "CPython-2.7.10": {
+                "binary": "/Users/user/pants/build-support/pants_dev_deps.venv/bin/python2.7",
+                "chroot": "/Users/user/pants/.pants.d/python-setup/chroots/e8da2c200f36ca0a1b8a60c12590a59209250b1a"
+            }
+        },
+        "default_interpreter": "CPython-2.7.10"
+    },
     "targets": {
         "examples/tests/java/org/pantsbuild/example/usethrift:usethrift": {
             "is_code_gen": false,
@@ -124,11 +133,32 @@ The following is an abbreviated export file from the command in the pants repo:
             ],
             "roots": [
                 {
-                    "source_root": "/Users/zundel/Src/pants/examples/tests/java/org/pantsbuild/example/usethrift",
+                    "source_root": "/Users/user/pants/examples/tests/java/org/pantsbuild/example/usethrift",
                     "package_prefix": "org.pantsbuild.example.usethrift"
                 }
             ]
         },
+        "examples/src/python/example/hello/greet:greet": {
+            "is_code_gen": false,
+            "python_interpreter": "CPython-2.7.10",
+            "target_type": "SOURCE",
+            "libraries": [],
+            "pants_target_type": "python_library",
+            "globs": {
+                "globs": [
+                    "examples/src/python/example/hello/greet/*.py"
+                ]
+            },
+            "targets": [
+                "3rdparty/python:ansicolors"
+            ],
+            "roots": [
+                {
+                    "source_root": "/Users/user/pants/examples/src/python/example/hello/greet",
+                    "package_prefix": "example.hello.greet"
+                }
+            ]
+        },
 
         ...
    }

tests/python/pants_test/backend/project_info/tasks/BUILD

@@ -86,6 +86,7 @@ python_tests(
   name = 'export_integration',
   sources = ['test_export_integration.py'],
   dependencies = [
+    '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/base:build_environment',
     'src/python/pants/ivy',
     'src/python/pants/util:contextutil',

tests/python/pants_test/backend/project_info/tasks/test_export.py

@@ -198,7 +198,7 @@ def test_version(self):
     result = get_json(self.execute_console_task(
       targets=[self.target('project_info:first')]
     ))
-    self.assertEqual('1.0.3', result['version'])
+    self.assertEqual('1.0.4', result['version'])
 
   def test_sources(self):
     result = get_json(self.execute_console_task(

tests/python/pants_test/backend/project_info/tasks/test_export_integration.py

@@ -8,6 +8,8 @@
 import json
 import os
 
+from twitter.common.collections import maybe_list
+
 from pants.base.build_environment import get_buildroot
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
@@ -21,7 +23,7 @@ class ExportIntegrationTest(PantsRunIntegrationTest):
 
   def run_export(self, test_target, workdir, load_libs=False, extra_args=None):
     export_out_file = os.path.join(workdir, 'export_out.txt')
-    args = ['export', '--output-file={out_file}'.format(out_file=export_out_file), test_target]
+    args = ['export', '--output-file={out_file}'.format(out_file=export_out_file)] + maybe_list(test_target)
     libs_args = ['--no-export-libraries'] if not load_libs else self._confs_args
     pants_run = self.run_pants_with_workdir(args + libs_args + (extra_args or []), workdir)
     self.assert_success(pants_run)
@@ -164,6 +166,7 @@ def test_distributions_and_platforms(self):
         ' "linux": [ "/usr/lib/jdk7", "/usr/lib/jdk8"]'
         '}'
       ])
+      self.assertFalse('python_setup' in json_data)
       target_name = 'examples/src/java/org/pantsbuild/example/hello/simple:simple'
       targets = json_data.get('targets')
       self.assertEquals('java7', targets[target_name]['platform'])
@@ -188,3 +191,28 @@ def test_distributions_and_platforms(self):
           }
         },
         json_data['jvm_platforms'])
+
+  def test_intellij_integration(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      targets = ['src/python/::', 'tests/python/pants_test:all', 'contrib/::']
+      excludes = [
+        '--exclude-target-regexp=.*go/examples.*',
+        '--exclude-target-regexp=.*scrooge/tests/thrift.*',
+        '--exclude-target-regexp=.*spindle/tests/thrift.*',
+        '--exclude-target-regexp=.*spindle/tests/jvm.*'
+      ]
+      json_data = self.run_export(targets, workdir, extra_args=excludes)
+
+      python_setup = json_data['python_setup']
+      self.assertIsNotNone(python_setup)
+      self.assertIsNotNone(python_setup['interpreters'])
+
+      default_interpreter = python_setup['default_interpreter']
+      self.assertIsNotNone(default_interpreter)
+      self.assertIsNotNone(python_setup['interpreters'][default_interpreter])
+      self.assertTrue(os.path.exists(python_setup['interpreters'][default_interpreter]['binary']))
+      self.assertTrue(os.path.exists(python_setup['interpreters'][default_interpreter]['chroot']))
+
+      core_target = json_data['targets']['src/python/pants/backend/core:core']
+      self.assertIsNotNone(core_target)
+      self.assertEquals(default_interpreter, core_target['python_interpreter'])