A contrib plugin to run the Kythe indexer on Java source. (pantsbuild…

…#4457)

BUILD.tools

@@ -35,3 +35,16 @@ jar_library(name = 'scrooge-linter',
               '3rdparty:thrift-0.6.1',
             ])
 
+
+# Google doesn't publish Kythe jars (yet?).  So we publish them to a custom repo
+# (https://github.com/benjyw/binhost) for now.  See build-support/ivy/ivysettings.xml
+# for more info.
+jar_library(
+  name='kythe-extractor',
+  jars = [jar(org='kythe', name='extractors/javac_extractor', rev='v0.0.26-snowchain3')]
+)
+
+jar_library(
+  name='kythe-indexer',
+  jars = [jar(org='kythe', name='indexers/java_indexer', rev='v0.0.26-snowchain3')]
+)

build-support/ivy/ivysettings.xml

@@ -12,6 +12,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
   <property name="m2.repo.artifact"
             value="${m2.repo.relpath}/[artifact](-[classifier])-[revision].[ext]"/>
   <property name="m2.repo.dir" value="${user.home}/.m2/repository" override="false"/>
+  <property name="kythe.artifact" value="[organization]-[revision]/[artifact].[ext]"/>
 
   <!-- This dance to set ANDROID_HOME seems more complex than it need be, but an absolute path is
        needed whether or not the ANDROID_HOME env var is set and just using the following does
@@ -63,6 +64,11 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
         <ivy pattern="${m2.repo.dir}/${m2.repo.pom}"/>
         <artifact pattern="${m2.repo.dir}/${m2.repo.artifact}"/>
       </filesystem>
+
+      <!-- Custom repo for Kythe jars, as Google don't currently publish them anywhere. -->
+      <url name="benjyw/binhost">
+        <artifact pattern="https://github.com/benjyw/binhost/raw/master/[organisation]/${kythe.artifact}" />
+      </url>
     </chain>
   </resolvers>
 </ivysettings>

contrib/kythe/src/python/pants/contrib/kythe/BUILD

@@ -0,0 +1,13 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+contrib_plugin(
+  name='plugin',
+  dependencies=[
+    'contrib/kythe/src/python/pants/contrib/kythe/tasks',
+    'src/python/pants/goal:task_registrar',
+  ],
+  distribution_name='pantsbuild.pants.contrib.kythe',
+  description='Kythe support for pants.',
+  register_goals=True,
+)

contrib/kythe/src/python/pants/contrib/kythe/register.py

@@ -0,0 +1,16 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.goal.task_registrar import TaskRegistrar as task
+
+from pants.contrib.kythe.tasks.extract_java import ExtractJava
+from pants.contrib.kythe.tasks.index_java import IndexJava
+
+
+def register_goals():
+  task(name='extract', action=ExtractJava).install('kythe')
+  task(name='index', action=IndexJava).install('kythe')

contrib/kythe/src/python/pants/contrib/kythe/tasks/BUILD

@@ -0,0 +1,13 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    'src/python/pants/backend/jvm/subsystems:shader',
+    'src/python/pants/backend/jvm/targets:jvm',
+    'src/python/pants/backend/jvm/tasks:jvm_tool_task_mixin',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/build_graph',
+    'src/python/pants/java/jar',
+  ]
+)

contrib/kythe/src/python/pants/contrib/kythe/tasks/extract_java.py

@@ -0,0 +1,113 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.backend.jvm.subsystems.jvm import JVM
+from pants.backend.jvm.subsystems.shader import Shader
+from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
+from pants.base.exceptions import TaskError
+
+from pants.contrib.kythe.tasks.indexable_java_targets import IndexableJavaTargets
+
+
+# Kythe requires various system properties to be set (sigh).  So we can't use nailgun.
+class ExtractJava(JvmToolTaskMixin):
+  _KYTHE_EXTRACTOR_MAIN = 'com.google.devtools.kythe.extractors.java.standalone.Javac8Wrapper'
+
+  cache_target_dirs = True
+
+  @classmethod
+  def implementation_version(cls):
+    # Bump this version to invalidate all past artifacts generated by this task.
+    return super(ExtractJava, cls).implementation_version() + [('KytheExtract', 6), ]
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(ExtractJava, cls).subsystem_dependencies() + (JVM.scoped(cls),)
+
+  @classmethod
+  def product_types(cls):
+    # TODO: Support indexpack files?
+    return ['kindex_files']
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    super(ExtractJava, cls).prepare(options, round_manager)
+    round_manager.require_data('runtime_classpath')
+    round_manager.require_data('zinc_args')
+
+  @classmethod
+  def register_options(cls, register):
+    super(ExtractJava, cls).register_options(register)
+    cls.register_jvm_tool(register,
+                          'kythe-extractor',
+                          custom_rules=[
+                            # These need to remain unshaded so that Kythe can interact with the
+                            # javac embedded in its jar.
+                            Shader.exclude_package('com.sun', recursive=True),
+                          ],
+                          main=cls._KYTHE_EXTRACTOR_MAIN)
+
+  def execute(self):
+    indexable_targets = IndexableJavaTargets.get(self.context)
+    targets_to_zinc_args = self.context.products.get_data('zinc_args')
+
+    with self.invalidated(indexable_targets, invalidate_dependents=True) as invalidation_check:
+      cp = self.tool_classpath('kythe-extractor')
+      for vt in invalidation_check.invalid_vts:
+        self.context.log.info('Kythe extracting from {}\n'.format(vt.target.address.spec))
+        javac_args = self._get_javac_args_from_zinc_args(targets_to_zinc_args[vt.target])
+        jvm_options = list(JVM.scoped_instance(self).get_jvm_options())
+        jvm_options.extend([
+          '-DKYTHE_CORPUS={}'.format(vt.target.address.spec),
+          '-DKYTHE_ROOT_DIRECTORY={}'.format(vt.target.target_base),
+          '-DKYTHE_OUTPUT_DIRECTORY={}'.format(vt.results_dir)
+        ])
+
+        result = self.dist.execute_java(
+          classpath=cp, main=self._KYTHE_EXTRACTOR_MAIN,
+          jvm_options=jvm_options, args=javac_args, workunit_name='kythe-extract')
+        if result != 0:
+          raise TaskError('java {main} ... exited non-zero ({result})'.format(
+            main=self._KYTHE_EXTRACTOR_MAIN, result=result))
+
+    for vt in invalidation_check.all_vts:
+      created_files = os.listdir(vt.results_dir)
+      if len(created_files) != 1:
+        raise TaskError('Expected a single .kindex file in {}. Got: {}.'.format(
+          vt.results_dir, ', '.join(created_files) if created_files else 'none'))
+      kindex_files = self.context.products.get_data('kindex_files', dict)
+      kindex_files[vt.target] = os.path.join(vt.results_dir, created_files[0])
+
+  @staticmethod
+  def _get_javac_args_from_zinc_args(zinc_args):
+    javac_args = []
+    i = iter(zinc_args)
+    arg = next(i, None)
+    output_dir = None
+    while arg is not None:
+      arg = arg.strip()
+      if arg in ['-d', '-cp', '-classpath']:
+        # These are passed through from zinc to javac.
+        javac_args.append(arg)
+        javac_args.append(next(i))
+        if arg == '-d':
+          output_dir = javac_args[-1]
+      elif arg.startswith('-C'):
+        javac_args.append(arg[2:])
+      elif arg.endswith('.java'):
+        javac_args.append(arg)
+      arg = next(i, None)
+    # Strip output dir from classpaths.  If we don't then javac will read annotation definitions
+    # from there instead of from the source files, which will cause the vnames to reflect the .class
+    # file instead of the .java file.
+    if output_dir:
+      for i, a in enumerate(javac_args):
+        if a in ['-cp', '-classpath']:
+          javac_args[i + 1] = ':'.join([p for p in javac_args[i + 1].split(':') if p != output_dir])
+    return javac_args

contrib/kythe/src/python/pants/contrib/kythe/tasks/index_java.py

@@ -0,0 +1,72 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.backend.jvm.tasks.nailgun_task import NailgunTask
+from pants.base.exceptions import TaskError
+from pants.base.workunit import WorkUnitLabel
+
+from pants.contrib.kythe.tasks.indexable_java_targets import IndexableJavaTargets
+
+
+class IndexJava(NailgunTask):
+  _KYTHE_INDEXER_MAIN = 'com.google.devtools.kythe.analyzers.java.JavaIndexer'
+
+  cache_target_dirs = True
+
+  @classmethod
+  def implementation_version(cls):
+    # Bump this version to invalidate all past artifacts generated by this task.
+    return super(IndexJava, cls).implementation_version() + [('IndexJava', 6), ]
+
+  @classmethod
+  def product_types(cls):
+    return ['kythe_entries_files']
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    super(IndexJava, cls).prepare(options, round_manager)
+    round_manager.require_data('kindex_files')
+
+  @classmethod
+  def register_options(cls, register):
+    super(IndexJava, cls).register_options(register)
+    register('--force', type=bool, fingerprint=True,
+             help='Re-index all targets, even if they are valid.')
+    cls.register_jvm_tool(register,
+                          'kythe-indexer',
+                          main=cls._KYTHE_INDEXER_MAIN)
+
+  def execute(self):
+    def entries_file(_vt):
+      return os.path.join(_vt.results_dir, 'index.entries')
+
+    indexable_targets = IndexableJavaTargets.get(self.context)
+
+    with self.invalidated(indexable_targets, invalidate_dependents=True) as invalidation_check:
+      kindex_files = self.context.products.get_data('kindex_files')
+      cp = self.tool_classpath('kythe-indexer')
+      vts_to_index = (invalidation_check.all_vts if self.get_options().force
+                      else invalidation_check.invalid_vts)
+
+      for vt in vts_to_index:
+        self.context.log.info('Kythe indexing {}'.format(vt.target.address.spec))
+        kindex_file = kindex_files.get(vt.target)
+        if not kindex_file:
+          raise TaskError('No .kindex file found for {}'.format(vt.target.address.spec))
+        args = [kindex_file, '--out', entries_file(vt)]
+        result = self.runjava(classpath=cp, main=self._KYTHE_INDEXER_MAIN,
+                              jvm_options=self.get_options().jvm_options,
+                              args=args, workunit_name='kythe-index',
+                              workunit_labels=[WorkUnitLabel.COMPILER])
+        if result != 0:
+          raise TaskError('java {main} ... exited non-zero ({result})'.format(
+            main=self._KYTHE_INDEXER_MAIN, result=result))
+
+    for vt in invalidation_check.all_vts:
+      self.context.products.get_data('kythe_entries_files', dict)[vt.target] = entries_file(vt)

contrib/kythe/src/python/pants/contrib/kythe/tasks/indexable_java_targets.py

@@ -0,0 +1,30 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.jvm.targets.jvm_target import JvmTarget
+from pants.build_graph.target_scopes import Scopes
+
+
+class IndexableJavaTargets(object):
+  """Determines which java targets Kythe should act on."""
+
+  @classmethod
+  def get(cls, context):
+    """Return the indexable targets in the given context.
+
+    Computes them lazily from the given context.  They are then fixed for the duration
+    of the run, even if this method is called again with a different context.
+    """
+    if not cls._targets:
+      # TODO: Should we index COMPILE scoped deps? E.g., annotations?
+      cls._targets = context.targets(
+        lambda t: isinstance(t, JvmTarget) and t.has_sources('.java'),
+        exclude_scopes=Scopes.COMPILE
+      )
+    return cls._targets
+
+  _targets = None

contrib/kythe/tests/python/pants_test/contrib/kythe/tasks/BUILD

@@ -0,0 +1,13 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+python_tests(
+  name = 'integration',
+  sources = globs('*_integration.py'),
+  dependencies=[
+    'tests/python/pants_test:int-test',
+  ],
+  tags={'integration'},
+  timeout=180,
+)

contrib/kythe/tests/python/pants_test/contrib/kythe/tasks/test_index_java_integration.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import glob
+import os
+
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class TestIndexJavaIntegration(PantsRunIntegrationTest):
+  def test_index_simple_java_code(self):
+    # Very simple test that we can run the extractor and indexer on some
+    # fairly trivial code without crashing, and that we produce something.
+    args = ['kythe', 'examples/src/java/org/pantsbuild/example/hello::']
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir(args, workdir)
+      self.assert_success(pants_run)
+      for tgt in ['examples.src.java.org.pantsbuild.example.hello.greet.greet',
+                  'examples.src.java.org.pantsbuild.example.hello.main.main-bin',
+                  'examples.src.java.org.pantsbuild.example.hello.simple.simple']:
+        kindex_glob = os.path.join(workdir,
+                                   'kythe/extract/current/{}/current/*.kindex'.format(tgt))
+        kindex_files = glob.glob(kindex_glob)
+        self.assertEquals(1, len(kindex_files))
+        kindex_file = kindex_files[0]
+        self.assertTrue(os.path.isfile(kindex_file))
+        self.assertGreater(os.path.getsize(kindex_file), 200)  # Make sure it's not trivial.
+
+        entries_path = os.path.join(workdir,
+                                    'kythe/index/current/{}/current/index.entries'.format(tgt))
+        self.assertTrue(os.path.isfile(entries_path))
+        self.assertGreater(os.path.getsize(entries_path), 1000)  # Make sure it's not trivial.

pants.ini

@@ -30,6 +30,7 @@ pythonpath: [
     "%(buildroot)s/contrib/findbugs/src/python",
     "%(buildroot)s/contrib/go/src/python",
     "%(buildroot)s/contrib/jax_ws/src/python",
+    "%(buildroot)s/contrib/kythe/src/python",
     "%(buildroot)s/contrib/node/src/python",
     "%(buildroot)s/contrib/python/src/python",
     "%(buildroot)s/contrib/scalajs/src/python",
@@ -49,6 +50,7 @@ backend_packages: +[
     "pants.contrib.findbugs",
     "pants.contrib.go",
     "pants.contrib.jax_ws",
+    "pants.contrib.kythe",
     "pants.contrib.scalajs",
     "pants.contrib.node",
     "pants.contrib.python.checks",

src/python/pants/backend/jvm/subsystems/java.py

@@ -12,6 +12,7 @@
 from pants.subsystem.subsystem import Subsystem
 
 
+# TODO: Sort out JVM compile config model: https://github.com/pantsbuild/pants/issues/4483.
 class Java(JvmToolMixin, ZincLanguageMixin, Subsystem):
   """A subsystem to encapsulate compile-time settings and features for the Java language.
 

src/python/pants/backend/jvm/subsystems/scala_platform.py

@@ -32,6 +32,7 @@
 scala_style_jar = JarDependency('org.scalastyle', 'scalastyle_2.11', '0.8.0')
 
 
+# TODO: Sort out JVM compile config model: https://github.com/pantsbuild/pants/issues/4483.
 class ScalaPlatform(JvmToolMixin, ZincLanguageMixin, Subsystem):
   """A scala platform.
 

src/python/pants/backend/jvm/tasks/jvm_compile/compile_context.py

@@ -77,13 +77,14 @@ class CompileContext(object):
   """
 
   def __init__(self, target, analysis_file, portable_analysis_file, classes_dir, jar_file,
-               log_file, sources, strict_deps):
+               log_file, zinc_args_file, sources, strict_deps):
     self.target = target
     self.analysis_file = analysis_file
     self.portable_analysis_file = portable_analysis_file
     self.classes_dir = classes_dir
     self.jar_file = jar_file
     self.log_file = log_file
+    self.zinc_args_file = zinc_args_file
     self.sources = sources
     self.strict_deps = strict_deps