In process java compilation in Zinc pantsbuild#1555

To avoid forking javac each time let's add nessesary JDK jars to zinc's classpath if JDK is presented.

Testing Done:
https://travis-ci.org/fkorotkov/pants/builds/62250441

Bugs closed: 1555

Reviewed at https://rbcommons.com/s/twitter/r/2206/

src/python/pants/backend/codegen/tasks/jaxb_gen.py

@@ -34,10 +34,9 @@ def __init__(self, *args, **kwargs):
     lang = 'java'
     if self.context.products.isrequired(lang):
       self.gen_langs.add(lang)
-    self.jar_location = os.path.join(Distribution.cached().home, '..', 'lib', 'tools.jar')
 
   def _compile_schema(self, args):
-    classpath = [self.jar_location]
+    classpath = Distribution.cached(jdk=True).find_libs(['tools.jar'])
     java_main = 'com.sun.tools.internal.xjc.Driver'
     return self.runjava(classpath=classpath, main=java_main, args=args, workunit_name='xjc')
 

src/python/pants/backend/jvm/tasks/jvm_compile/scala/scala_compile.py

@@ -19,6 +19,7 @@
 from pants.backend.jvm.tasks.jvm_compile.scala.zinc_utils import ZincUtils
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
+from pants.java.distribution.distribution import Distribution
 from pants.option.options import Options
 from pants.util.contextutil import open_zip
 from pants.util.dirutil import safe_open
@@ -90,7 +91,17 @@ def create_analysis_tools(self):
     return AnalysisTools(self.context.java_home, ZincAnalysisParser(), ZincAnalysis)
 
   def zinc_classpath(self):
-    return self.tool_classpath('zinc')
+    # Zinc takes advantage of tools.jar if it's presented in classpath.
+    # For example com.sun.tools.javac.Main is used for in process java compilation.
+    def locate_tools_jar():
+      try:
+        return Distribution.cached(jdk=True).find_libs(['tools.jar'])
+      except Distribution.Error:
+        self.context.log.info('Failed to locate tools.jar. '
+                              'Install a JDK to increase performance of Zinc.')
+        return []
+
+    return self.tool_classpath('zinc') + locate_tools_jar()
 
   def compiler_classpath(self):
     return ScalaPlatform.global_instance().compiler_classpath(self.context.products)

src/python/pants/java/distribution/distribution.py

@@ -150,6 +150,24 @@ def version(self):
     """
     return self._get_version(self.java)
 
+  def find_libs(self, names):
+    """Looks for jars in lib folder.
+
+    :param list names: jar file names
+    :return: list of paths to requested libraries
+    :raises: `Distribution.Error` if any of the jars could not be found.
+    """
+    def collect_existing_libs():
+      # self._bin_path points to bin folder.
+      jdk_root = os.path.dirname(self._bin_path)
+      for name in names:
+        lib_path = os.path.join(jdk_root, 'lib', name)
+        if not os.path.exists(lib_path):
+          raise Distribution.Error('Failed to locate {} library'.format(name))
+        yield lib_path
+
+    return list(collect_existing_libs())
+
   @property
   def home(self):
     """Returns the distribution JAVA_HOME."""

tests/python/pants_test/backend/jvm/tasks/jvm_compile/base_compile_integration_test.py

@@ -23,17 +23,7 @@ def do_test_compile(self, target, strategy,
     """
     with temporary_dir(root_dir=self.workdir_root()) as workdir:
       for i in xrange(0, iterations):
-        args = [
-            'compile',
-            '--compile-apt-strategy={}'.format(strategy),
-            '--compile-java-strategy={}'.format(strategy),
-            '--compile-scala-strategy={}'.format(strategy),
-            target,
-          ] + (extra_args if extra_args else [])
-        # Clean-all on the first iteration.
-        if i == 0:
-          args.insert(0, 'clean-all')
-        pants_run = self.run_pants_with_workdir(args, workdir)
+        pants_run = self.run_test_compile(workdir, target, strategy, clean_all=(i == 0), extra_args=extra_args)
         if expect_failure:
           self.assert_failure(pants_run)
         else:
@@ -51,6 +41,20 @@ def do_test_compile(self, target, strategy,
                           'Failed to find the following compiled files: {}'.format(to_find))
         yield found
 
+  def run_test_compile(self, workdir, target, strategy, clean_all=False, extra_args=None):
+    args = [
+        'compile',
+        '--compile-apt-strategy={}'.format(strategy),
+        '--compile-java-strategy={}'.format(strategy),
+        '--compile-scala-strategy={}'.format(strategy),
+        '--compile-zinc-java-strategy={}'.format(strategy),
+        target,
+      ] + (extra_args if extra_args else [])
+    # Clean-all on the first iteration.
+    if clean_all:
+      args.insert(0, 'clean-all')
+    return self.run_pants_with_workdir(args, workdir)
+
   def get_only(self, found, name):
     files = found[name]
     self.assertEqual(1, len(files))

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_zinc_compile_integration.py

@@ -5,15 +5,25 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+from pants.util.contextutil import temporary_dir
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
 from pants_test.backend.jvm.tasks.jvm_compile.utils import provide_compile_strategies
 
 
 class JvmExamplesCompileIntegrationTest(BaseCompileIT):
   @provide_compile_strategies
   def test_java_src_zinc_compile(self, strategy):
-    self.do_test_compile('examples/src/java/::', strategy, extra_args='--compile-zinc-java-enabled')
+    self.do_test_compile('examples/src/java/::', strategy, extra_args=['--compile-zinc-java-enabled'])
 
   @provide_compile_strategies
   def test_java_tests_zinc_compile(self, strategy):
-    self.do_test_compile('examples/tests/java/::', strategy, extra_args='--compile-zinc-java-enabled')
+    self.do_test_compile('examples/tests/java/::', strategy, extra_args=['--compile-zinc-java-enabled'])
+
+  @provide_compile_strategies
+  def test_in_process(self, strategy):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      pants_run = self.run_test_compile(
+        workdir, 'examples/tests/java/::', strategy, extra_args=['--compile-zinc-java-enabled', '-ldebug']
+      )
+      self.assertTrue('Attempting to call javac directly' in pants_run.stdout_data)
+      self.assertFalse('Forking javac' in pants_run.stdout_data)

tests/python/pants_test/java/distribution/test_distribution.py

@@ -46,16 +46,24 @@ def env(self, **kwargs):
       yield
 
   @contextmanager
-  def distribution(self, files=None, executables=None):
-    with temporary_dir() as jdk:
+  def distribution(self, files=None, executables=None, libs=None):
+    with temporary_dir() as jdk_root:
+      jdk_bin_dir = os.path.join(jdk_root, 'bin')
+      os.mkdir(jdk_bin_dir)
       for f in maybe_list(files or ()):
-        touch(os.path.join(jdk, f))
+        touch(os.path.join(jdk_bin_dir, f))
       for exe in maybe_list(executables or (), expected_type=self.EXE):
-        path = os.path.join(jdk, exe.name)
+        path = os.path.join(jdk_bin_dir, exe.name)
         with safe_open(path, 'w') as fp:
           fp.write(exe.contents or '')
         chmod_plus_x(path)
-      yield jdk
+
+      jdk_lib_dir = os.path.join(jdk_root, 'lib')
+      os.mkdir(jdk_lib_dir)
+      for f in maybe_list(libs or ()):
+        touch(os.path.join(jdk_lib_dir, f))
+
+      yield jdk_bin_dir
 
   def setUp(self):
     super(MockDistributionTest, self).setUp()
@@ -108,6 +116,14 @@ def test_validated_binary(self):
     with self.distribution(executables=[self.exe('java'), self.exe('jar')]) as jdk:
       Distribution(bin_path=jdk).binary('jar')
 
+  def test_validated_library(self):
+    with pytest.raises(Distribution.Error):
+      with self.distribution(executables=self.exe('java')) as jdk:
+        Distribution(bin_path=jdk).find_libs(['tools.jar'])
+
+    with self.distribution(executables=self.exe('java'), libs='tools.jar') as jdk:
+      Distribution(bin_path=jdk).find_libs(['tools.jar'])
+
   def test_locate(self):
 
     with pytest.raises(Distribution.Error):