Add android_library target and unpack_library task

This review is part 1 of splitting review pantsbuild#2040. This adds
android_library and android_dependency targets as well as an
unpack_libraries task. This is not integrated with the rest
of the android backend, that will be part 2.

An android_dependency can be a jar or aar library artifact, in
a repo or from the SDK. Those artifacts have to be unpacked and then
repacked into the final android application.

Each library is unpacked exactly once. The includes/excludes
will be handled in the repacking. That keeps us from having to repeatedly
unpack common dependencies. They all have a common base so this
treats the unpacked source as this common starting point.

If the library is an aar file, it needs to add targets to the
graph based on the aar's contents, as seen in UnpackLibraries.

The follow up will integrate the tasks/targets into the other tasks
and finish up adding the testing harness. If you'd like to see how
it runs or what the BUILD files will look like, you can check pantsbuild#2040.

references pantsbuild#1390

Testing Done:
Passes local ci.sh and travis.

Bugs closed: 1800

Reviewed at https://rbcommons.com/s/twitter/r/2467/

examples/src/android/example/AndroidManifest.xml → examples/src/android/hello/AndroidManifest.xml

@@ -4,7 +4,7 @@ Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 Licensed under the Apache License, Version 2.0 (see LICENSE).
 -->
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="org.pantsbuild.example.hello"
+    package="org.pantsbuild.examples.hello"
     android:versionCode="1"
     android:versionName="1.0" >
 
@@ -18,7 +18,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
         android:label="@string/appName">
         android:theme="@style/AppTheme" >
         <activity
-            android:name="org.pantsbuild.example.hello.HelloWorld"
+            android:name="org.pantsbuild.examples.hello.HelloWorld"
             android:configChanges="orientation|keyboardHidden|screenSize"
             android:label="@string/appName">
             android:theme="@style/FullscreenTheme" >

examples/src/android/example/src/com/pants/examples/hello/HelloWorld.java → examples/src/android/hello/src/org/pantsbuild/examples/hello/HelloWorld.java

@@ -2,7 +2,7 @@
 // Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-package org.pantsbuild.example.hello;
+package org.pantsbuild.examples.hello;
 
 import android.app.Activity;
 import android.content.res.Resources;

src/python/pants/backend/android/android_manifest_parser.py

@@ -27,21 +27,18 @@ def parse_manifest(cls, manifest_path):
     """
     try:
       manifest = XmlParser.from_file(manifest_path)
-      target_sdk = manifest.get_attribute('uses-sdk', 'android:targetSdkVersion')
       package_name = manifest.get_attribute('manifest', 'package')
     except XmlParser.XmlError as e:
       raise cls.BadManifestError("AndroidManifest.xml parsing error: {}".format(e))
-    app_name = manifest.get_optional_attribute('activity', 'android:name')
+    target_sdk = manifest.get_optional_attribute('uses-sdk', 'android:targetSdkVersion')
 
-    return AndroidManifest(manifest.xml_path, target_sdk, package_name, app_name=app_name)
+    return AndroidManifest(manifest.xml_path, target_sdk, package_name)
 
 
 class AndroidManifest(object):
   """Object to represent an Android manifest."""
 
-  def __init__(self, path, target_sdk, package_name, app_name=None):
+  def __init__(self, path, target_sdk, package_name):
     self.path = path
     self.target_sdk = target_sdk
     self.package_name = package_name
-    # Can be None, so tasks should use target.app_name which checks this but has a backup value.
-    self.app_name = app_name

src/python/pants/backend/android/register.py

@@ -6,11 +6,14 @@
                         unicode_literals, with_statement)
 
 from pants.backend.android.targets.android_binary import AndroidBinary
+from pants.backend.android.targets.android_dependency import AndroidDependency
+from pants.backend.android.targets.android_library import AndroidLibrary
 from pants.backend.android.targets.android_resources import AndroidResources
 from pants.backend.android.tasks.aapt_builder import AaptBuilder
 from pants.backend.android.tasks.aapt_gen import AaptGen
 from pants.backend.android.tasks.dx_compile import DxCompile
 from pants.backend.android.tasks.sign_apk import SignApkTask
+from pants.backend.android.tasks.unpack_libraries import UnpackLibraries
 from pants.backend.android.tasks.zipalign import Zipalign
 from pants.base.build_file_aliases import BuildFileAliases
 from pants.goal.task_registrar import TaskRegistrar as task
@@ -20,11 +23,14 @@ def build_file_aliases():
   return BuildFileAliases.create(
     targets={
       'android_binary': AndroidBinary,
+      'android_dependency': AndroidDependency,
+      'android_library': AndroidLibrary,
       'android_resources': AndroidResources,
     }
   )
 
 def register_goals():
+  task(name='unpack-libs', action=UnpackLibraries).install('unpack-jars')
   task(name='aapt', action=AaptGen).install('gen')
   task(name='dex', action=DxCompile).install('binary')
   task(name='apk', action=AaptBuilder).install()

src/python/pants/backend/android/targets/BUILD

@@ -13,16 +13,20 @@ python_library(
   name = 'android',
   sources = [
     'android_binary.py',
+    'android_dependency.py',
+    'android_library.py',
     'android_resources.py',
     'android_target.py',
   ],
   dependencies = [
     'src/python/pants/backend/jvm/targets:jvm',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:exceptions',
+    'src/python/pants/base:payload',
     'src/python/pants/base:payload_field',
     'src/python/pants/base:target',
     'src/python/pants/backend/android:android_manifest_parser',
-    'src/python/pants/backend/core/targets:common'
+    'src/python/pants/backend/core/targets:common',
+    'src/python/pants/util:memo',
   ],
 )

src/python/pants/backend/android/targets/android_binary.py

@@ -6,7 +6,20 @@
                         unicode_literals, with_statement)
 
 from pants.backend.android.targets.android_target import AndroidTarget
+from pants.base.exceptions import TargetDefinitionException
+from pants.util.memo import memoized_property
 
 
 class AndroidBinary(AndroidTarget):
   """Produces an Android binary."""
+
+  def __init__(self, *args, **kwargs):
+    super(AndroidBinary, self).__init__(*args, **kwargs)
+
+  @memoized_property
+  def target_sdk(self):
+    """Return the SDK version to use when compiling this target."""
+    if not self.manifest.target_sdk:
+      raise TargetDefinitionException(self, "AndroidBinary targets must declare targetSdkVersion "
+                                            "in the AndroidManifest.xml.")
+    return self.manifest.target_sdk

src/python/pants/backend/android/targets/android_dependency.py

@@ -0,0 +1,12 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.jvm.targets.jar_library import JarLibrary
+
+
+class AndroidDependency(JarLibrary):
+  """A set of artifacts that may be depended upon by Android targets."""

src/python/pants/backend/android/targets/android_library.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.backend.android.android_manifest_parser import AndroidManifestParser
+from pants.backend.android.targets.android_target import AndroidTarget
+from pants.backend.jvm.targets.import_jars_mixin import ImportJarsMixin
+from pants.base.payload import Payload
+from pants.base.payload_field import PrimitiveField
+from pants.util.memo import memoized_property
+
+
+class AndroidLibrary(ImportJarsMixin, AndroidTarget):
+  """Android library projects that access Android API or Android resources.
+  """
+  def __init__(self, payload=None, libraries=None,
+               include_patterns=None, exclude_patterns=None, **kwargs):
+    """
+    :param list libraries: List of addresses of `android_dependency <#android_dependency>`_
+      targets.
+    :param list include_patterns: fileset patterns to include from the archive
+    :param list exclude_patterns: fileset patterns to exclude from the archive
+    """
+
+    # TODO(mateor) Perhaps add a BUILD file attribute to force archive type: one of (jar, aar).
+    payload = payload or Payload()
+    payload.add_fields({
+      'library_specs': PrimitiveField(libraries or ())
+    })
+    self.libraries = libraries
+    self.include_patterns = include_patterns or []
+    self.exclude_patterns = exclude_patterns or []
+
+    super(AndroidLibrary, self).__init__(payload=payload, **kwargs)
+
+  @property
+  def imported_jar_library_specs(self):
+    """List of JarLibrary specs to import.
+
+    Required to implement the ImportJarsMixin.
+    """
+    return self.payload.library_specs
+
+  @memoized_property
+  def manifest(self):
+    """The manifest of the AndroidLibrary, if one exists."""
+    # Libraries may not have a manifest, so self.manifest can be None for android_libraries.
+    if self._manifest_file is None:
+      return None
+    else:
+      manifest_path = os.path.join(self._spec_path, self._manifest_file)
+    return AndroidManifestParser.parse_manifest(manifest_path)

src/python/pants/backend/android/targets/android_resources.py

@@ -17,7 +17,6 @@ class AndroidResources(AndroidTarget):
   def __init__(self,
                resource_dir=None,
                **kwargs):
-    #TODO(mateor) change resource_dir from string into list
     """
     :param string resource_dir: path/to/directory containing Android resource files,
      often named 'res'.

src/python/pants/backend/android/targets/android_target.py

@@ -10,12 +10,12 @@
 from pants.backend.android.android_manifest_parser import AndroidManifestParser
 from pants.backend.jvm.targets.jvm_target import JvmTarget
 from pants.base.exceptions import TargetDefinitionException
+from pants.util.memo import memoized_property
 
 
 class AndroidTarget(JvmTarget):
   """A base class for all Android targets."""
 
-
   def __init__(self,
                address=None,
                # TODO (mateor) add support for minSDk
@@ -35,32 +35,18 @@ def __init__(self,
     self.build_tools_version = build_tools_version
     self._spec_path = address.spec_path
 
-    self._manifest_path = manifest
-    self._manifest = None
-    self._app_name = None
+    self._manifest_file = manifest
 
-  @property
+  @memoized_property
   def manifest(self):
     """Return an AndroidManifest object made from a manifest by AndroidManifestParser."""
 
-    # For both gradle and ant layouts, AndroidManifest is conventionally at top-level.
-    # I would recommend users still explicitly define a 'manifest' in android BUILD files.
-    if self._manifest is None:
-      # If there was no 'manifest' field in the BUILD file, try to find one with the default value.
-      if self._manifest_path is None:
-        self._manifest_path = 'AndroidManifest.xml'
-      manifest = os.path.join(self._spec_path, self._manifest_path)
-      if not os.path.isfile(manifest):
-        raise TargetDefinitionException(self, "There is no AndroidManifest.xml at path {0}. Please "
-                                              "declare a 'manifest' field with its relative "
-                                              "path.".format(manifest))
-      self._manifest = AndroidManifestParser.parse_manifest(manifest)
-    return self._manifest
-
-  @property
-  def app_name(self):
-    """Return application name from the target's manifest or target.name if that cannot be found."""
-    if self._app_name is None:
-      app_name = self.manifest.app_name or self.name
-      self._app_name = app_name.split(".")[-1]
-    return self._app_name
+    # If there was no 'manifest' field in the BUILD file, try to find one with the default value.
+    if self._manifest_file is None:
+      self._manifest_file = 'AndroidManifest.xml'
+    manifest_path = os.path.join(self._spec_path, self._manifest_file)
+    if not os.path.isfile(manifest_path):
+      raise TargetDefinitionException(self, "There is no AndroidManifest.xml at path {0}. Please "
+                                            "declare a 'manifest' field with its relative "
+                                            "path.".format(manifest_path))
+    return AndroidManifestParser.parse_manifest(manifest_path)

src/python/pants/backend/android/tasks/BUILD

@@ -11,6 +11,7 @@ python_library(
     ':aapt_task',
     ':dx_compile',
     ':sign_apk',
+    ':unpack_libraries',
     ':zipalign',
   ],
 )
@@ -92,6 +93,20 @@ python_library(
   ],
 )
 
+python_library(
+  name = 'unpack_libraries',
+  sources = ['unpack_libraries.py'],
+  dependencies = [
+    'src/python/pants/backend/android/targets:android',
+    'src/python/pants/backend/core/tasks:common',
+    'src/python/pants/backend/jvm/targets:java',
+    'src/python/pants/base:address',
+    'src/python/pants/base:build_environment',
+    'src/python/pants/base:fingerprint_strategy',
+    'src/python/pants/fs',
+  ],
+)
+
 python_library(
   name = 'zipalign',
   sources = ['zipalign.py'],

src/python/pants/backend/android/tasks/aapt_builder.py

@@ -61,7 +61,7 @@ def render_args(self, target, resource_dir, inputs):
     args.extend(['-I', self.android_jar_tool(target.manifest.target_sdk)])
     args.extend(['--ignore-assets', self.ignored_assets])
     args.extend(['-F', os.path.join(self.workdir,
-                                    '{0}.unsigned.apk'.format(target.app_name))])
+                                    '{0}.unsigned.apk'.format(target.manifest.package_name))])
     args.extend(inputs)
     logger.debug('Executing: {0}'.format(' '.join(args)))
     return args
@@ -95,5 +95,5 @@ def gather_resources(target):
           if returncode:
             raise TaskError('Android aapt tool exited non-zero: {0}'.format(returncode))
     for target in targets:
-      apk_name = '{0}.unsigned.apk'.format(target.app_name)
+      apk_name = '{0}.unsigned.apk'.format(target.manifest.package_name)
       self.context.products.get('apk').add(target, self.workdir).append(apk_name)

src/python/pants/backend/android/tasks/sign_apk.py

@@ -61,7 +61,7 @@ def setup_default_config(cls, path):
   @classmethod
   def signed_package_name(cls, target, build_type):
     """Get package name with 'build_type', a string KeyResolver mandates is in (debug, release)."""
-    return '{0}.{1}.signed.apk'.format(target.app_name, build_type)
+    return '{0}.{1}.signed.apk'.format(target.manifest.package_name, build_type)
 
   def __init__(self, *args, **kwargs):
     super(SignApkTask, self).__init__(*args, **kwargs)