added sources and javadocs to export goal output

Testing Done:
added a test

https://travis-ci.org/fkorotkov/pants/builds/54885417

Reviewed at https://rbcommons.com/s/twitter/r/1936/

src/python/pants/backend/project_info/register.py

@@ -30,7 +30,9 @@ def register_goals():
   task(name='ensime', action=EnsimeGen).install().with_description(
       'Create an Ensime project from the given targets.')
 
-  task(name='export', action=Export).install().with_description("Export project information for targets in JSON format.")
+  task(name='export', action=Export).install().with_description(
+    'Export project information for targets in JSON format. '
+    'Use with resolve goal to get detailed information about libraries.')
 
   task(name='depmap', action=Depmap).install().with_description("Depict the target's dependencies.")
 

src/python/pants/backend/project_info/tasks/BUILD

@@ -72,6 +72,7 @@ python_library(
   name = 'export',
   sources = ['export.py'],
   dependencies = [
+    ':ide_gen',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/backend/core/targets:all',
     'src/python/pants/backend/core/tasks:console_task',

src/python/pants/backend/project_info/tasks/export.py

@@ -13,10 +13,10 @@
 
 from pants.backend.core.targets.resources import Resources
 from pants.backend.core.tasks.console_task import ConsoleTask
-from pants.backend.jvm.targets.jar_dependency import JarDependency
 from pants.backend.jvm.targets.jar_library import JarLibrary
 from pants.backend.jvm.targets.jvm_binary import JvmApp
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
+from pants.backend.project_info.tasks.ide_gen import IdeGen
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 
@@ -58,12 +58,6 @@ def _address(address):
   @classmethod
   def register_options(cls, register):
     super(Export, cls).register_options(register)
-    register('--project-info', default=True, action='store_true',
-             deprecated_version='0.0.31',
-             deprecated_hint='This option is a no-op. Project info format is always enabled for '
-                             'the export goal.',
-             help='Produces a json object with info about the target, including source roots, '
-                  'dependencies, and paths to libraries for their targets and dependencies.')
     register('--formatted', default=True, action='store_false',
              help='Causes output to be a single line of JSON.')
 
@@ -78,13 +72,6 @@ def __init__(self, *args, **kwargs):
     self.target_aliases_map = None
 
   def console_output(self, targets):
-    if len(self.context.target_roots) == 0:
-      raise TaskError("One or more target addresses are required.")
-    output = self.project_info_output(targets)
-    for line in output:
-      yield line
-
-  def project_info_output(self, targets):
     targets_map = {}
     resource_target_map = {}
     ivy_jar_products = self.context.products.get_data('ivy_jar_products') or {}
@@ -176,11 +163,9 @@ def get_transitive_jars(jar_lib):
   def _resolve_jars_info(self):
     mapping = defaultdict(list)
     jar_data = self.context.products.get_data('ivy_jar_products')
-    if not jar_data:
-      return mapping
-    for dep in jar_data['default']:
-      for module in dep.modules_by_ref.values():
-        mapping[self._jar_id(module.ref)] = [artifact.path for artifact in module.artifacts]
+    jar_infos = IdeGen.get_jar_infos(ivy_products=jar_data, confs=['default', 'sources', 'javadoc'])
+    for jar, paths in jar_infos.iteritems():
+      mapping[self._jar_id(jar)] = paths
     return mapping
 
   def _get_pants_target_alias(self, pants_target_type):

src/python/pants/backend/project_info/tasks/ide_gen.py

@@ -276,26 +276,29 @@ def map_internal_jars(self, targets):
 
           self._project.internal_jars.add(ClasspathEntry(cp_jar, source_jar=cp_source_jar))
 
-  def _get_jar_paths(self, confs=None):
+  @staticmethod
+  def get_jar_infos(ivy_products, confs=None):
     """Returns a list of dicts containing the paths of various jar file resources.
 
     Keys include 'default' (normal jar path), 'sources' (path to source jar), and 'javadoc'
     (path to doc jar). None of them are guaranteed to be present, but 'sources' and 'javadoc'
     will never be present if 'default' isn't.
 
+    :param ivy_products: ivy_jar_products data from a context
     :param confs: List of key types to return (eg ['default', 'sources']). Just returns 'default' if
       left unspecified.
+    :returns {dict}
     """
-    ivy_products = self.context.products.get_data('ivy_jar_products')
     classpath_maps = defaultdict(dict)
-    for conf, info_group in ivy_products.items():
-      if conf not in confs:
-        continue # We don't care about it.
-      for info in info_group:
-        for module in info.modules_by_ref.values():
-          for artifact in module.artifacts:
-            classpath_maps[(module.ref.org, module.ref.name, module.ref.rev,)][conf] = artifact.path
-    return classpath_maps.values()
+    if ivy_products:
+      for conf, info_group in ivy_products.items():
+        if conf not in confs:
+          continue # We don't care about it.
+        for info in info_group:
+          for module in info.modules_by_ref.values():
+            for artifact in module.artifacts:
+              classpath_maps[module.ref][conf] = artifact.path
+    return classpath_maps
 
   def map_external_jars(self):
     external_jar_dir = os.path.join(self.gen_project_workdir, 'external-libs')
@@ -308,7 +311,8 @@ def map_external_jars(self):
     safe_mkdir(external_javadoc_jar_dir, clean=True)
 
     confs = ['default', 'sources', 'javadoc']
-    for entry in self._get_jar_paths(confs=confs):
+    jar_paths = self.get_jar_infos(self.context.products.get_data('ivy_jar_products'), confs)
+    for entry in jar_paths.values():
       jar = entry.get('default')
       if jar:
         cp_jar = os.path.join(external_jar_dir, os.path.basename(jar))

tests/python/pants_test/tasks/test_export_integration.py

@@ -16,10 +16,16 @@
 
 
 class ExportIntegrationTest(PantsRunIntegrationTest):
+  _resolve_args = [
+    'resolve',
+    '--resolve-ivy-confs=default',
+    '--resolve-ivy-confs=sources',
+    '--resolve-ivy-confs=javadoc',
+  ]
 
-  def run_export(self, test_target, workdir):
+  def run_export(self, test_target, workdir, extra_args = list()):
     export_out_file = os.path.join(workdir, 'export_out.txt')
-    pants_run = self.run_pants_with_workdir([
+    pants_run = self.run_pants_with_workdir(extra_args + [
         'export',
         '--output-file={out_file}'.format(out_file=export_out_file),
         test_target],
@@ -51,18 +57,39 @@ def test_export_json_transitive_jar(self):
   def test_export_jar_path(self):
     with temporary_dir(root_dir=self.workdir_root()) as workdir:
       test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
-      json_data = self.run_export(test_target, workdir)
+      json_data = self.run_export(test_target, workdir, self._resolve_args)
       # Hack because Bootstrapper.instance() reads config from cache. Will go away after we plumb
       # options into IvyUtil properly.
       Config.cache(Config.load())
       ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir
-      self.assertEquals(json_data.get('libraries').get('commons-lang:commons-lang:2.5'),
-                        [os.path.join(ivy_cache_dir,
-                                      'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
+      common_lang_lib_info = json_data.get('libraries').get('commons-lang:commons-lang:2.5')
+      self.assertIsNotNone(common_lang_lib_info)
+      self.assertEquals(
+        common_lang_lib_info.get('default'),
+        os.path.join(ivy_cache_dir, 'commons-lang/commons-lang/jars/commons-lang-2.5.jar')
+      )
+      self.assertEquals(
+        common_lang_lib_info.get('javadoc'),
+        os.path.join(ivy_cache_dir, 'commons-lang/commons-lang/javadocs/commons-lang-2.5-javadoc.jar')
+      )
+      self.assertEquals(
+        common_lang_lib_info.get('sources'),
+        os.path.join(ivy_cache_dir, 'commons-lang/commons-lang/sources/commons-lang-2.5-sources.jar')
+      )
 
   def test_dep_map_for_java_sources(self):
     with temporary_dir(root_dir=self.workdir_root()) as workdir:
       test_target = 'examples/src/scala/com/pants/example/scala_with_java_sources'
       json_data = self.run_export(test_target, workdir)
       targets = json_data.get('targets')
       self.assertIn('examples/src/java/com/pants/examples/java_sources:java_sources', targets)
+
+  def test_sources_and_javadocs(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      test_target = 'examples/src/scala/com/pants/example/scala_with_java_sources'
+      json_data = self.run_export(test_target, workdir, self._resolve_args)
+      scala_lang_lib = json_data.get('libraries').get('org.scala-lang:scala-library:2.10.4')
+      self.assertIsNotNone(scala_lang_lib)
+      self.assertIsNotNone(scala_lang_lib['default'])
+      self.assertIsNotNone(scala_lang_lib['sources'])
+      self.assertIsNotNone(scala_lang_lib['javadoc'])