Squashed commit of the Enabling local publish for pants build. Issue-…

…46: Cherrypick 64b54c04 from twitter/commons

pants.ini

@@ -303,6 +303,8 @@ jvm_args: ['-Xmx1g', '-XX:MaxPermSize=256m']
 [jar-create]
 workdir: %(pants_workdir)s/jars
 
+[jar-publish]
+ivy_settings: %(pants_supportdir)s/ivy/ivysettings.xml
 
 [args-resource-mapper]
 classdirs: [ '%(java_workdir)s/classes', '%(scala_workdir)s' ]

src/python/pants/authentication/BUILD

@@ -0,0 +1,8 @@
+from pants.authentication.netrc_util import Netrc
+
+netrc = Netrc()
+
+credentials(
+  name = 'netrc',
+  username=netrc.getusername,
+  password=netrc.getpassword)

src/python/pants/authentication/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).

src/python/pants/authentication/netrc_util.py

@@ -0,0 +1,44 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import print_function
+
+import collections
+import os
+
+from netrc import netrc as NetrcDb, NetrcParseError
+
+from pants.tasks.task_error import TaskError
+
+
+class Netrc(object):
+
+  def __init__(self):
+    self._login = collections.defaultdict(lambda: None)
+    self._password = collections.defaultdict(lambda: None)
+
+  def getusername(self, repository):
+    self._ensure_loaded()
+    return self._login[repository]
+
+  def getpassword(self, repository):
+    self._ensure_loaded()
+    return self._password[repository]
+
+  def _ensure_loaded(self):
+    if not self._login and not self._password:
+      db = os.path.expanduser('~/.netrc')
+      if not os.path.exists(db):
+        raise TaskError('A ~/.netrc file is required to authenticate')
+      try:
+        db = NetrcDb(db)
+        for host, value in db.hosts.items():
+          auth = db.authenticators(host)
+          if auth:
+            login, _, password = auth
+            self._login[host] = login
+            self._password[host] = password
+        if len(self._login) == 0:
+          raise TaskError('Found no usable authentication blocks for twitter in ~/.netrc')
+      except NetrcParseError as e:
+        raise TaskError('Problem parsing ~/.netrc: %s' % e)

src/python/pants/commands/goal.py

@@ -495,6 +495,7 @@ def cleanup(self):
 from pants.tasks.filedeps import FileDeps
 from pants.tasks.ivy_resolve import IvyResolve
 from pants.tasks.jar_create import JarCreate
+from pants.tasks.jar_publish import JarPublish
 from pants.tasks.javadoc_gen import JavadocGen
 from pants.tasks.junit_run import JUnitRun
 from pants.tasks.jvm_compile.java.java_compile import JavaCompile
@@ -793,22 +794,24 @@ def __init__(self, context, output_dir=None, confs=None):
                                          active=False)
 
 
-class JarCreateGoal(JarCreate):
-  def __init__(self, context):
-    super(JarCreateGoal, self).__init__(context, False)
-
 goal(name='javadoc_publish',
-     action=JavadocJarShim).install('jar')
+     action=JavadocJarShim).install('publish')
 goal(name='scaladoc_publish',
-     action=ScaladocJarShim).install('jar')
+     action=ScaladocJarShim).install('publish')
 goal(name='jar',
-     action=JarCreateGoal,
+     action=JarCreate,
      dependencies=['compile', 'resources', 'bootstrap']).install('jar').with_description('Create one or more jars.')
 goal(name='check_published_deps',
      action=CheckPublishedDeps
 ).install('check_published_deps').with_description(
   'Find references to outdated artifacts published from this BUILD tree.')
 
+goal(name='jar_create_publish',
+     action=JarCreate,
+     dependencies=['compile', 'resources']).install('publish')
+
+goal(name='publish',
+     action=JarPublish).install('publish').with_description('Publish one or more artifacts.')
 
 goal(name='junit',
      action=JUnitRun,

src/python/pants/tasks/jar_create.py

@@ -20,6 +20,7 @@
 from pants.tasks.scaladoc_gen import scaladoc
 
 
+
 DEFAULT_CONFS = ['default']
 
 
@@ -75,12 +76,15 @@ def setup_parser(cls, option_group, args, mkflag):
                             dest='jar_create_sources', default=False,
                             action='callback', callback=mkflag.set_bool,
                             help='[%default] Create source jars.')
+    #TODO tdesai: Think about a better way to set defaults per goal basis.
+    javadoc_defaults = True if option_group.title.split(':')[0] == 'publish' else False
     option_group.add_option(mkflag('javadoc'), mkflag('javadoc', negate=True),
-                            dest='jar_create_javadoc', default=False,
+                            dest='jar_create_javadoc',
+                            default=javadoc_defaults,
                             action='callback', callback=mkflag.set_bool,
                             help='[%default] Create javadoc jars.')
 
-  def __init__(self, context, jar_javadoc=False):
+  def __init__(self, context):
     Task.__init__(self, context)
 
     options = context.options
@@ -99,11 +103,11 @@ def __init__(self, context, jar_javadoc=False):
 
     definitely_create_javadoc = options.jar_create_javadoc or products.isrequired('javadoc_jars')
     definitely_dont_create_javadoc = options.jar_create_javadoc is False
-    create_javadoc = jar_javadoc and options.jar_create_javadoc is None
+    create_javadoc = options.jar_create_javadoc
     if definitely_create_javadoc and definitely_dont_create_javadoc:
       self.context.log.warn('javadoc jars are required but you have requested they not be created, '
                             'creating anyway')
-    self.jar_javadoc = (True  if definitely_create_javadoc      else
+    self.jar_javadoc = (True if definitely_create_javadoc else
                         False if definitely_dont_create_javadoc else
                         create_javadoc)
     if self.jar_javadoc:

src/python/pants/tasks/jar_publish.py

@@ -115,6 +115,7 @@ def as_jar(internal_target):
     # the graph
     dependencies = OrderedDict()
     internal_codegen = {}
+    configurations = set()
     for dep in target_internal_dependencies(target):
       jar = as_jar(dep)
       dependencies[(jar.org, jar.name)] = self.internaldep(jar, dep)
@@ -123,7 +124,11 @@ def as_jar(internal_target):
     for jar in target.jar_dependencies:
       if jar.rev:
         dependencies[(jar.org, jar.name)] = self.jardep(jar)
-    target_jar = self.internaldep(as_jar(target)).extend(dependencies=dependencies.values())
+        configurations |= set(jar._configurations)
+
+    target_jar = self.internaldep(
+                     as_jar(target),
+                     configurations=list(configurations)).extend(dependencies=dependencies.values())
 
     template_kwargs = self.templateargs(target_jar, confs)
     with safe_open(path, 'w') as output:
@@ -137,7 +142,7 @@ def templateargs(self, target_jar, confs=None):
     """
     raise NotImplementedError()
 
-  def internaldep(self, jar_dependency, dep=None):
+  def internaldep(self, jar_dependency, dep=None, configurations=None):
     """
       Subclasses must return a template data for the given internal target (provided in jar
       dependency form).
@@ -166,7 +171,7 @@ def jardep(self, jar):
         scope='compile',
         excludes=[self.create_exclude(exclude) for exclude in jar.excludes if exclude.name])
 
-  def internaldep(self, jar_dependency, dep=None):
+  def internaldep(self, jar_dependency, dep=None, configurations=None):
     return self.jardep(jar_dependency)
 
 
@@ -196,10 +201,10 @@ def _jardep(self, jar, transitive=True, configurations='default'):
   def jardep(self, jar):
     return self._jardep(jar,
         transitive=jar.transitive,
-        configurations=';'.join(jar._configurations))
+        configurations=jar._configurations)
 
-  def internaldep(self, jar_dependency, dep=None):
-    return self._jardep(jar_dependency)
+  def internaldep(self, jar_dependency, dep=None, configurations=None):
+    return self._jardep(jar_dependency, configurations=configurations)
 
 
 def coordinate(org, name, rev=None):
@@ -363,6 +368,8 @@ def __init__(self, context, scm=None):
       self.snapshot = context.options.jar_publish_local_snapshot
     else:
       self.repos = context.config.getdict(JarPublish._CONFIG_SECTION, 'repos')
+      if not self.repos:
+        raise TaskError("This repo is not yet set for publishing to the world! Please re-run with --publish-local")
       for repo, data in self.repos.items():
         auth = data.get('auth')
         if auth:
@@ -605,7 +612,7 @@ def publish(ivyxml_path):
             try:
               ivy = Bootstrapper.default_ivy()
               ivy.execute(jvm_options=jvm_args, args=args,
-                          workunit_factory=self.context.new_workunit, workunit_name = 'jar-publish')
+                          workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
             except (Bootstrapper.Error, Ivy.Error) as e:
               raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
 
@@ -682,7 +689,7 @@ def exported_targets(self):
       def get_synthetic(lang, target):
         mappings = self.context.products.get(lang).get(target)
         if mappings:
-          for generated in mappings.itervalues():
+          for key, generated in mappings.items():
             for synthetic in generated:
               yield synthetic
 

src/python/pants/tasks/jvmdoc_gen.py

@@ -141,7 +141,7 @@ def generate_execute(self, targets, language_predicate, create_jvmdoc_command):
           'Cannot provide %s target mappings for combined output' % self._jvmdoc.product_type)
     elif catalog or self.active:
       def docable(target):
-        return language_predicate(target) and (self._include_codegen or not is_codegen(target))
+        return language_predicate(target) and (self._include_codegen or not target.is_codegen)
 
       with self.invalidated(filter(docable, targets)) as invalidation_check:
         safe_mkdir(self._output_dir)

src/python/pants/tasks/templates/ivy_resolve/ivy.mustache

@@ -18,11 +18,11 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
    />
 
   {{#lib.configurations?}}
-  <configurations>
-    {{#lib.configurations}}
-    <conf name="{{.}}"/>
-    {{/lib.configurations}}
-  </configurations>
+    <configurations>
+      {{#lib.configurations}}
+      <conf name="{{.}}"/>
+      {{/lib.configurations}}
+     </configurations>
   {{/lib.configurations?}}
 
   {{#lib.publications?}}
@@ -52,7 +52,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
                 {{#mutable}}changing="true"{{/mutable}}
     >
       {{#configurations}}
-      <conf name="{{.}}" mapped="{{.}}"/>
+      <conf name="{{.}}"/>
       {{/configurations}}
       {{#artifacts}}
       <artifact