Adding the ability to pull in a Maven artifact and extract its conten…

…ts for use to feeding into another task.

UnpackedJars  (unpacked_jars) is a target that extracts files from a jar artifact.
The UnpackJars task does the extraction work.
DeferredSourcesMapper is a task that feeds some unpacked files into another task's sources= attribute.
Initially supported by java_protobuf_library targets.

Adds FromTarget (from_target) and which is used to re-direct the sources attribute to one of these UnpackedJars target
Also adds DeferredSourcesField which is a SourcesField that doesn't know
the exact list of sources when the graph is initially built.

Initial Design doc: https://docs.google.com/a/squareup.com/document/d/1CeKmYBRDq_Agn_YO-Nn6Ek4amGbQnJpV3PceB0_tSsU/edit

Testing Done:
Added some unit tests, an example and an integration test.
CI running at https://travis-ci.org/pantsbuild/pants/builds/47172787

Bugs closed: 720

Reviewed at https://rbcommons.com/s/twitter/r/1210/

BUILD

@@ -26,7 +26,7 @@ source_root('testprojects/tests/scala', page, junit_tests, scala_library, scala_
 source_root('examples/src/android', page, android_resources, android_binary)
 source_root('examples/src/antlr', page, java_antlr_library, python_antlr_library)
 source_root('examples/src/java', annotation_processor, jvm_binary, java_library, page)
-source_root('examples/src/protobuf', java_protobuf_library, jar_library, page)
+source_root('examples/src/protobuf', java_protobuf_library, jar_library, unpacked_jars, page)
 source_root('examples/src/python', page, python_binary, python_library, resources)
 source_root('examples/src/resources', page, resources, jaxb_library)
 source_root('examples/src/scala', jvm_binary, page, scala_library, benchmark)

examples/src/java/com/pants/examples/protobuf/unpacked_jars/BUILD

@@ -0,0 +1,11 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+jvm_binary(name='unpacked_jars',
+  basename='protobuf-unpacked-jars-example',
+  source='ExampleProtobufExternalArchive.java',
+  main='com.pants.examples.protobuf.unpacked_jars.ExampleProtobufExternalArchive',
+  dependencies=[
+    'examples/src/protobuf/com/pants/examples/unpacked_jars'
+  ],
+)

examples/src/java/com/pants/examples/protobuf/unpacked_jars/ExampleProtobufExternalArchive.java

@@ -0,0 +1,17 @@
+// Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+package com.pants.examples.protobuf.unpacked_jars;
+
+import com.squareup.testing.protolib.External;
+
+class ExampleProtobufExternalArchive {
+  private ExampleProtobufExternalArchive() {
+  }
+
+  public static void main(String[] args) {
+    External.ExternalMessage message = External.ExternalMessage.newBuilder().setMessageType(1)
+      .setMessageContent("Hello World!").build();
+    System.out.println("Message is: " + message.getMessageContent());
+  }
+}

examples/src/protobuf/com/pants/examples/unpacked_jars/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+java_protobuf_library(name='unpacked_jars',
+  sources=from_target(':external-source'),
+)
+
+unpacked_jars(name='external-source',
+  libraries=[':external-source-jars'],
+  include_patterns=[
+    'com/squareup/testing/**/*.proto',
+  ],
+ )
+
+jar_library(name='external-source-jars',
+  jars=[
+    jar(org='com.squareup.testing.protolib', name='protolib-external-test', rev='0.0.2'),
+  ],
+)

src/python/pants/backend/codegen/tasks/protobuf_gen.py

@@ -79,6 +79,7 @@ def __init__(self, *args, **kwargs):
   def prepare(self, round_manager):
     super(ProtobufGen, self).prepare(round_manager)
     round_manager.require_data('ivy_imports')
+    round_manager.require_data('deferred_sources')
   # TODO https://github.com/pantsbuild/pants/issues/604 prep finish
 
   def resolve_deps(self, key, default=None):
@@ -193,7 +194,7 @@ def add_to_gentargets(target):
       add_to_gentargets,
       postorder=True)
     sources_by_base = OrderedDict()
-    # TODO(Eric Ayers) Extract this logic for general use? When using jar_source_set it is needed
+    # TODO(Eric Ayers) Extract this logic for general use? When using unpacked_jars it is needed
     # to get the correct source root for paths outside the current BUILD tree.
     for target in gentargets:
       for source in target.sources_relative_to_buildroot():

src/python/pants/backend/core/BUILD

@@ -16,11 +16,15 @@ python_library(
 
 python_library(
   name = 'core',
-  sources = ['wrapped_globs.py'],
+  sources = [
+    'from_target.py',
+    'wrapped_globs.py',
+  ],
   dependencies = [
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
     'src/python/pants/backend/core/targets:all',
     'src/python/pants/backend/core/tasks:all',
+    'src/python/pants/base:address',
     'src/python/pants/base:build_environment',
     'src/python/pants/util:dirutil',
   ]

src/python/pants/backend/core/from_target.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
+                        print_function, unicode_literals)
+
+import six
+
+from pants.base.address import Addresses
+
+
+class FromTarget(object):
+  """Used in a BUILD file to redirect the value of the sources= attribute to another target.
+  """
+  class ExpectedAddressError(Exception):
+    """Thrown if an object that is not an address is added to an import attribute.
+    """
+
+  def __init__(self, parse_context):
+    """
+    :param ParseContext parse_context: build file context
+    """
+    self._parse_context = parse_context
+
+  def __call__(self, address):
+    """Expects a string representing an address."""
+    if not isinstance(address, six.string_types):
+      raise self.ExpectedAddressError("Expected string address argument, got type {type}"
+                                     .format(type(address)))
+    return Addresses(addresses=[address], rel_path=self._parse_context.rel_path)

src/python/pants/backend/core/register.py

@@ -17,6 +17,7 @@
 from pants.backend.core.tasks.changed_target_goals import CompileChanged, TestChanged
 from pants.backend.core.tasks.clean import Cleaner, Invalidator
 from pants.backend.core.tasks.confluence_publish import ConfluencePublish
+from pants.backend.core.tasks.deferred_sources_mapper import DeferredSourcesMapper
 from pants.backend.core.tasks.dependees import ReverseDepmap
 from pants.backend.core.tasks.filemap import Filemap
 from pants.backend.core.tasks.filter import Filter
@@ -34,6 +35,7 @@
 from pants.backend.core.tasks.sorttargets import SortTargets
 from pants.backend.core.tasks.targets_help import TargetsHelp
 from pants.backend.core.tasks.what_changed import WhatChanged
+from pants.backend.core.from_target import FromTarget
 from pants.backend.core.wrapped_globs import Globs, RGlobs, ZGlobs
 from pants.base.build_environment import get_buildroot, pants_version
 from pants.base.build_file_aliases import BuildFileAliases
@@ -85,6 +87,7 @@ def build_file_aliases():
     context_aware_object_factories={
       'buildfile_path': BuildFilePath,
       'globs': Globs,
+      'from_target': FromTarget,
       'rglobs': RGlobs,
       'source_root': SourceRoot.factory,
       'zglobs': ZGlobs,
@@ -181,3 +184,6 @@ def execute(self):
   task(name='test', action=NoopTest).install('test')
   task(name='test-changed', action=TestChanged).install().with_description(
     'Test changed targets.')
+
+  task(name='deferred-sources', action=DeferredSourcesMapper).install().with_description(
+    'Map unpacked sources from archives.')

src/python/pants/backend/core/tasks/BUILD

@@ -11,6 +11,7 @@ target(
     ':common',
     ':confluence_publish',
     ':console_task',
+    ':deferred_sources_mapper',
     ':dependees',
     ':filemap',
     ':filter',
@@ -47,6 +48,9 @@ python_library(
   sources = ['build_lint.py'],
   dependencies = [
     ':common',
+    'src/python/pants/base:address_lookup_error',
+    'src/python/pants/base:payload_field',
+    'src/python/pants/base:source_root',
   ],
 )
 
@@ -107,6 +111,14 @@ python_library(
   ],
 )
 
+python_library(
+  name = 'deferred_sources_mapper',
+  sources= ['deferred_sources_mapper.py'],
+  dependencies = [
+    ':common',
+  ],
+)
+
 python_library(
   name = 'dependees',
   sources = ['dependees.py'],

src/python/pants/backend/core/tasks/deferred_sources_mapper.py

@@ -0,0 +1,77 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
+                        print_function, unicode_literals)
+
+import logging
+
+from pants.backend.core.tasks.task import Task
+from pants.base.address_lookup_error import AddressLookupError
+from pants.base.payload_field import DeferredSourcesField
+from pants.base.source_root import SourceRoot
+
+
+logger = logging.getLogger(__name__)
+
+
+class DeferredSourcesMapper(Task):
+  """Map DeferredSorucesFields to files that produce product 'unpacked_archives', like UnpackJars
+
+  If you want a task to be able to map sources like this, make it require the  'deferred_sources'
+  product.
+  """
+
+  class SourcesTargetLookupError(AddressLookupError):
+    """Raised when the referenced target cannot be found in the build graph"""
+    pass
+
+  class NoUnpackedSourcesError(AddressLookupError):
+    """Raised when there are no files found unpacked from the archive"""
+    pass
+
+  def __init__(self, *args, **kwargs):
+    super(DeferredSourcesMapper, self).__init__(*args, **kwargs)
+
+  def prepare(self, round_manager):
+    super(DeferredSourcesMapper, self).prepare(round_manager)
+    round_manager.require_data('unpacked_archives')
+
+  @classmethod
+  def product_types(cls):
+    """
+    Declare product produced by this task
+
+    deferred_sources does not have any data associated with it. Downstream tasks can
+    depend on it just make sure that this task completes first.
+    :return:
+    """
+    return ['deferred_sources']
+
+  def execute(self):
+    deferred_sources_fields = []
+    def find_deferred_sources_fields(target):
+      for name, payload_field in target.payload.fields:
+        if isinstance(payload_field, DeferredSourcesField):
+          deferred_sources_fields.append((target, name, payload_field))
+    addresses = [target.address for target in self.context.targets()]
+    self.context.build_graph.walk_transitive_dependency_graph(addresses,
+                                                              find_deferred_sources_fields)
+
+    unpacked_sources = self.context.products.get_data('unpacked_archives')
+    for (target, name, payload_field) in deferred_sources_fields:
+      sources_target = self.context.build_graph.get_target(payload_field.address)
+      if not sources_target:
+        raise self.SourcesTargetLookupError(
+          "Couldn't find {sources_spec} referenced from {target} field {name} in build graph"
+          .format(sources_spec=payload_field.address.spec, target=target.address.spec, name=name))
+      if not sources_target in unpacked_sources:
+        raise self.NoUnpackedSourcesError(
+          "Target {sources_spec} referenced from {target} field {name} did not unpack any sources"
+          .format(spec=sources_target.address.spec, target=target.address.spec, name=name))
+      sources, rel_unpack_dir = unpacked_sources[sources_target]
+      SourceRoot.register_mutable(rel_unpack_dir)
+      payload_field.populate(sources, rel_unpack_dir)
+

src/python/pants/backend/jvm/ivy_utils.py

@@ -13,7 +13,7 @@
 import threading
 import xml
 
-from twitter.common.collections import OrderedDict, OrderedSet
+from twitter.common.collections import OrderedDict, OrderedSet, maybe_list
 
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
@@ -364,10 +364,11 @@ def mapjars(self, genmap, target, executor, workunit_factory=None, jars=None):
                    '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
       '-symlink',
     ]
+    confs = target.payload.get_field_value('configurations') or []
     self.exec_ivy(mapdir,
                   [target],
                   ivyargs,
-                  confs=target.payload.configurations,
+                  confs=maybe_list(confs),
                   ivy=Bootstrapper.default_ivy(executor),
                   workunit_factory=workunit_factory,
                   workunit_name='map-jars',

src/python/pants/backend/jvm/register.py

@@ -12,6 +12,7 @@
 from pants.backend.jvm.targets.benchmark import Benchmark
 from pants.backend.jvm.targets.credentials import Credentials
 from pants.backend.jvm.targets.exclude import Exclude
+from pants.backend.jvm.targets.unpacked_jars import UnpackedJars
 from pants.backend.jvm.targets.jar_dependency import JarDependency
 from pants.backend.jvm.targets.jar_library import JarLibrary
 from pants.backend.jvm.targets.java_agent import JavaAgent
@@ -54,6 +55,7 @@
 from pants.backend.jvm.tasks.scala_repl import ScalaRepl
 from pants.backend.jvm.tasks.scaladoc_gen import ScaladocGen
 from pants.backend.jvm.tasks.specs_run import SpecsRun
+from pants.backend.jvm.tasks.unpack_jars import UnpackJars
 from pants.base.build_file_aliases import BuildFileAliases
 from pants.goal.task_registrar import TaskRegistrar as task
 from pants.goal.goal import Goal
@@ -66,6 +68,7 @@ def build_file_aliases():
       'benchmark': Benchmark,
       'credentials': Credentials,
       'jar_library': JarLibrary,
+      'unpacked_jars' : UnpackedJars,
       'java_agent': JavaAgent,
       'java_library': JavaLibrary,
       'java_tests': JavaTests,
@@ -111,6 +114,9 @@ def register_goals():
 
   task(name='ivy-imports', action=IvyImports).install('imports')
 
+  task(name='unpack-jars', action=UnpackJars).install().with_description(
+    'Unpack artifacts specified by unpacked_jars() targets.')
+
   # Compilation.
 
   # AnnotationProcessors are java targets, but we need to force them into their own compilation

src/python/pants/backend/jvm/targets/BUILD

@@ -15,6 +15,7 @@ python_library(
     'credentials.py',
     'exclude.py',
     'exportable_jvm_library.py',
+    'unpacked_jars.py',
     'jar_dependency.py',
     'jar_library.py',
     'jarable.py',

src/python/pants/backend/jvm/targets/jvm_target.py

@@ -5,15 +5,10 @@
 from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
                         print_function, unicode_literals)
 
-import six
-
 from pants.backend.core.targets.resources import Resources
 from pants.backend.jvm.targets.exclude import Exclude
-from pants.base.address import SyntheticAddress
 from pants.base.payload import Payload
-from pants.base.payload_field import (ConfigurationsField,
-                                      ExcludesField,
-                                      SourcesField)
+from pants.base.payload_field import ConfigurationsField, ExcludesField
 from pants.base.target import Target
 from pants.backend.jvm.targets.jar_library import JarLibrary
 from pants.backend.jvm.targets.jarable import Jarable
@@ -32,6 +27,7 @@ def __init__(self,
                resources=None,
                configurations=None,
                no_cache=False,
+               build_graph=None,
                **kwargs):
     """
     :param configurations: One or more ivy configurations to resolve for this target.
@@ -48,15 +44,15 @@ def __init__(self,
       sources_rel_path = address.spec_path
     payload = payload or Payload()
     payload.add_fields({
-      'sources': SourcesField(sources=self.assert_list(sources),
-                              sources_rel_path=sources_rel_path),
+      'sources': self.create_sources_field(sources, sources_rel_path, address, build_graph),
       'provides': provides,
       'excludes': ExcludesField(self.assert_list(excludes, expected_type=Exclude)),
       'configurations': ConfigurationsField(self.assert_list(configurations)),
     })
     self._resource_specs = self.assert_list(resources)
 
-    super(JvmTarget, self).__init__(address=address, payload=payload, **kwargs)
+    super(JvmTarget, self).__init__(address=address, payload=payload, build_graph=build_graph,
+                                    **kwargs)
     self.add_labels('jvm')
     if no_cache:
       self.add_labels('no_cache')