Add transitive jar dependencies to depmap project info goal for intel…

…lij plugin.

Add transitive jar dependencies to depmap goal for Pants Intellij plugin

When creating projects using the plugin, the depmap showed direct dependencies for 3rdparty jar libraries.
This change add the trasitive dependency jars to the depmap output.
The transitive jars are required to correctly configure intellij modules.

Testing Done:
yes.

Added integration tests but need to verify the contents.

PR created here:
pantsbuild#626
{Coverage increased (+0.04%) when pulling 09790ec on add_transitive_jar_deps into ad7d09d on master.}

Locally tested
./pants tests/python/pants_test/tasks:depmap_integration

Reviewed at https://rbcommons.com/s/twitter/r/1047/

src/python/pants/backend/jvm/ivy_utils.py

@@ -43,6 +43,14 @@ def add_module(self, module):
     for caller in module.callers:
       self.deps_by_caller[caller].add(module.ref)
 
+  def get_jars_for_ivy_module(self, jar):
+    ref = IvyModuleRef(jar.org, jar.name, jar.rev)
+    deps = OrderedSet()
+    for dep in self.deps_by_caller.get(ref, []):
+      deps.add(dep)
+      deps.update(self.get_jars_for_ivy_module(dep))
+    return deps
+
 
 class IvyUtils(object):
   IVY_TEMPLATE_PACKAGE_NAME = __name__

src/python/pants/backend/jvm/tasks/depmap.py

@@ -9,9 +9,12 @@
 import json
 import os
 
+from twitter.common.collections import OrderedSet
+
 from pants.backend.core.tasks.console_task import ConsoleTask
 from pants.backend.core.targets.dependencies import Dependencies
 from pants.backend.core.targets.resources import Resources
+from pants.backend.jvm.ivy_utils import IvyUtils
 from pants.backend.jvm.targets.jar_dependency import JarDependency
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.base.build_environment import get_buildroot
@@ -119,6 +122,9 @@ def __init__(self, *args, **kwargs):
     self.separator = self.context.options.depmap_separator
     self.project_info = self.context.options.depmap_is_project_info
     self.format = self.context.options.depmap_is_formatted
+    self._ivy_utils = IvyUtils(config=self.context.config,
+                               options=self.context.options,
+                               log=self.context.log)
 
   def console_output(self, targets):
     if len(self.context.target_roots) == 0:
@@ -244,6 +250,7 @@ def output_deps(outputted, dep, parent=None):
   def project_info_output(self, targets):
     targets_map = {}
     resource_target_map = {}
+    ivy_info = self._ivy_utils.parse_xml_report(targets, 'default')
 
     def process_target(current_target):
       """
@@ -253,43 +260,58 @@ def get_target_type(target):
         if target.is_test:
           return Depmap.SourceRootTypes.TEST
         else:
-          if isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test:
+          if (isinstance(target, Resources) and
+              target in resource_target_map and
+              resource_target_map[target].is_test):
             return Depmap.SourceRootTypes.TEST_RESOURCE
           elif isinstance(target, Resources):
             return Depmap.SourceRootTypes.RESOURCE
           else:
             return Depmap.SourceRootTypes.SOURCE
 
+      def get_transitive_jars(jar_lib):
+        if not ivy_info:
+          return OrderedSet()
+        transitive_jars = OrderedSet()
+        for jar in jar_lib.jar_dependencies:
+          transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar))
+        return transitive_jars
+
       info = {
         'targets': [],
         'libraries': [],
         'roots': [],
-        'target_type': get_target_type(current_target)
+        'target_type': get_target_type(current_target),
+        'is_code_gen': current_target.is_codegen
       }
 
+      target_libraries = set()
+      if current_target.is_jar_library:
+        target_libraries = get_transitive_jars(current_target)
       for dep in current_target.dependencies:
-        if dep.is_java or dep.is_jar_library or dep.is_jvm or dep.is_scala or dep.is_scalac_plugin:
-          info['targets'].append(self._address(dep.address))
+        info['targets'].append(self._address(dep.address))
         if dep.is_jar_library:
           for jar in dep.jar_dependencies:
-            info['libraries'].append(self._jar_id(jar))
+            target_libraries.add(jar)
+          # Add all the jars pulled in by this jar_library
+          target_libraries.update(get_transitive_jars(dep))
         if isinstance(dep, Resources):
-          info['targets'].append(self._address(dep.address))
           resource_target_map[dep] = current_target
 
-      java_sources_targets = list(current_target.java_sources) if isinstance(current_target, ScalaLibrary) else list()
+      java_sources_targets = list(current_target.java_sources) if isinstance(current_target,
+                                                                             ScalaLibrary) else []
       """
       :type java_sources_targets:list[pants.base.target.Target]
       """
 
       roots = set(itertools.chain(
         *[self._source_roots_for_target(t) for t in java_sources_targets + [current_target]]
       ))
-
       info['roots'] = map(lambda (source_root, package_prefix): {
         'source_root': source_root,
         'package_prefix': package_prefix
       }, roots)
+      info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
       targets_map[self._address(current_target.address)] = info
 
     for target in targets:

tests/python/pants_test/tasks/BUILD

@@ -74,6 +74,7 @@ python_test_suite(
 python_test_suite(
   name = 'integration',
   dependencies = [
+    ':depmap_integration',
     ':eclipse_integration',
     ':ensime_integration',
     ':idea_integration',
@@ -277,6 +278,16 @@ python_tests(
   ]
 )
 
+python_tests(
+  name = 'depmap_integration',
+  sources = ['test_depmap_integration.py'],
+  dependencies = [
+    'src/python/pants/ivy:ivy',
+    'src/python/pants/util:contextutil',
+    'tests/python/pants_test:int-test',
+  ],
+)
+
 python_tests(
   name = 'detect_duplicates',
   sources = ['test_detect_duplicates.py'],

tests/python/pants_test/tasks/test_depmap_integration.py

@@ -0,0 +1,76 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
+                        print_function, unicode_literals)
+
+import json
+import os
+
+from pants.util.contextutil import temporary_dir
+
+from pants.ivy.bootstrapper import Bootstrapper
+from pants.base.build_environment import get_buildroot
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class DepmapIntegrationTest(PantsRunIntegrationTest):
+
+  def _assert_run_success(self, pants_run):
+    self.assertEquals(pants_run.returncode, self.PANTS_SUCCESS_CODE,
+                      'goal depmap expected success, got {0}\n'
+                      'got stderr:\n{1}\n'
+                      'got stdout:\n{2}\n'.format(pants_run.returncode,
+                                                  pants_run.stderr_data,
+                                                  pants_run.stdout_data))
+
+  def run_depmap_project_info(self, test_target, workdir):
+    depmap_out_file = '{workdir}/depmap_out.txt'.format(workdir=workdir)
+    pants_run = self.run_pants_with_workdir(
+      ['goal', 'resolve', 'depmap', test_target, '--depmap-project-info',
+      '--depmap-output-file={out_file}'.format(out_file=depmap_out_file)], workdir)
+    self._assert_run_success(pants_run)
+    self.assertTrue(os.path.exists(depmap_out_file),
+                      msg='Could not find depmap output file in {out_file}'
+                        .format(out_file=depmap_out_file))
+    with open(depmap_out_file) as json_file:
+      json_data = json.load(json_file)
+      return json_data
+
+  def test_depmap_code_gen(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
+      json_data = self.run_depmap_project_info(test_target, workdir)
+      thrift_target_name = 'examples.src.thrift.com.pants.examples.precipitation.precipitation-java'
+      codegen_target = os.path.join(os.path.relpath(workdir, get_buildroot()),
+                                  'gen/thrift/combined/gen-java:%s' % thrift_target_name)
+      self.assertTrue(codegen_target in json_data.get('targets'))
+
+  def test_depmap_json_transitive_jar(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
+      json_data = self.run_depmap_project_info(test_target, workdir)
+      targets = json_data.get('targets')
+      self.assertTrue('org.hamcrest:hamcrest-core:1.3' in targets[test_target]['libraries'])
+
+  def test_depmap_jar_path(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
+      json_data = self.run_depmap_project_info(test_target, workdir)
+      ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir
+      self.assertEquals(json_data.get('libraries').get('commons-lang:commons-lang:2.5'),
+                      [os.path.join(ivy_cache_dir,
+                                    'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
+
+  def test_depmap_without_resolve(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      depmap_out_file = '{workdir}/depmap_out.txt'.format(workdir=workdir)
+      pants_run = self.run_pants_with_workdir(
+        ['goal', 'depmap', 'testprojects/src/java/com/pants/testproject/unicode/main',
+         '--depmap-project-info',
+         '--depmap-output-file={out_file}'.format(out_file=depmap_out_file)], workdir)
+      self._assert_run_success(pants_run)
+      self.assertTrue(os.path.exists(depmap_out_file),
+                      msg='Could not find depmap output file {out_file}'
+                           .format(out_file=depmap_out_file))