Finish splitting up the codegen backend.

All files in the old codegen backend are now forwarding stubs
to the per-IDL, per-output-language backends.

A future change will actually switch the pants repo's registration
to those new backends.  Once we're happy with that we can encourage
all other repos to follow suit.

contrib/scrooge/src/python/pants/contrib/scrooge/tasks/BUILD

@@ -51,7 +51,7 @@ python_library(
   name='java_thrift_library_fingerprint_strategy',
   sources=['java_thrift_library_fingerprint_strategy.py'],
   dependencies=[
-    'src/python/pants/backend/codegen/targets:java',
+    'src/python/pants/backend/codegen/thrift/java',
     'src/python/pants/base:fingerprint_strategy',
   ],
 )

src/python/pants/backend/codegen/BUILD

@@ -4,12 +4,15 @@
 
 python_library(
   name = 'plugin',
-  sources = ['__init__.py', 'register.py'],
   dependencies = [
-    'src/python/pants/backend/codegen/tasks:all',
-    'src/python/pants/backend/codegen/targets:java',
-    'src/python/pants/backend/codegen/targets:python',
+    'src/python/pants/backend/codegen/antlr/java',
+    'src/python/pants/backend/codegen/antlr/python',
+    'src/python/pants/backend/codegen/jaxb',
+    'src/python/pants/backend/codegen/protobuf/java',
+    'src/python/pants/backend/codegen/ragel/java',
     'src/python/pants/backend/codegen/thrift/java',
+    'src/python/pants/backend/codegen/thrift/python',
+    'src/python/pants/backend/codegen/wire/java',
     'src/python/pants/build_graph',
     'src/python/pants/goal:task_registrar',
   ]

src/python/pants/backend/codegen/antlr/java/BUILD

@@ -0,0 +1,15 @@
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies = [
+    'src/python/pants/backend/jvm/targets:java',
+    'src/python/pants/backend/jvm/targets:jvm',
+    'src/python/pants/backend/jvm/tasks:nailgun_task',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/java:util',
+    'src/python/pants/option',
+    'src/python/pants/task',
+    'src/python/pants/util:dirutil',
+  ],
+)

src/python/pants/backend/codegen/antlr/java/antlr_gen.py

@@ -0,0 +1,157 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import logging
+import os
+import re
+
+from pants.backend.codegen.antlr.java.java_antlr_library import JavaAntlrLibrary
+from pants.backend.jvm.targets.jar_dependency import JarDependency
+from pants.backend.jvm.targets.java_library import JavaLibrary
+from pants.backend.jvm.tasks.nailgun_task import NailgunTask
+from pants.base.exceptions import TaskError
+from pants.task.simple_codegen_task import SimpleCodegenTask
+from pants.util.dirutil import safe_mkdir, safe_walk
+
+
+logger = logging.getLogger(__name__)
+
+
+def antlr4_jar(name):
+  return JarDependency(org='org.antlr', name=name, rev='4.1')
+
+
+_DEFAULT_ANTLR_DEPS = {
+  'antlr3': ('//:antlr-3.4', [JarDependency(org='org.antlr', name='antlr', rev='3.4')]),
+  'antlr4': ('//:antlr-4', [antlr4_jar(name='antlr4'),
+                            antlr4_jar(name='antlr4-runtime')])
+}
+
+
+class AntlrGen(SimpleCodegenTask, NailgunTask):
+
+  class AmbiguousPackageError(TaskError):
+    """Raised when a java package cannot be unambiguously determined for a JavaAntlrLibrary."""
+
+  def find_sources(self, target, target_dir):
+    sources = super(AntlrGen, self).find_sources(target, target_dir)
+    return [source for source in sources if source.endswith('.java')]
+
+  @classmethod
+  def register_options(cls, register):
+    super(AntlrGen, cls).register_options(register)
+    for key, (classpath_spec, classpath) in _DEFAULT_ANTLR_DEPS.items():
+      cls.register_jvm_tool(register, key, classpath=classpath, classpath_spec=classpath_spec)
+
+  def is_gentarget(self, target):
+    return isinstance(target, JavaAntlrLibrary)
+
+  def synthetic_target_type(self, target):
+    return JavaLibrary
+
+  def execute_codegen(self, target, target_workdir):
+    args = ['-o', target_workdir]
+    compiler = target.compiler
+    if target.package is None:
+      java_package = self._get_sources_package(target)
+    else:
+      java_package = target.package
+
+    if compiler == 'antlr3':
+      if target.package is not None:
+        logger.warn("The 'package' attribute is not supported for antlr3 and will be ignored.")
+      java_main = 'org.antlr.Tool'
+    elif compiler == 'antlr4':
+      args.append('-visitor')  # Generate Parse Tree Visitor As Well
+      # Note that this assumes that there is no package set in the antlr file itself,
+      # which is considered an ANTLR best practice.
+      args.append('-package')
+      args.append(java_package)
+      java_main = 'org.antlr.v4.Tool'
+
+    antlr_classpath = self.tool_classpath(compiler)
+    sources = self._calculate_sources([target])
+    args.extend(sources)
+    result = self.runjava(classpath=antlr_classpath, main=java_main, args=args,
+                          workunit_name='antlr')
+    if result != 0:
+      raise TaskError('java {} ... exited non-zero ({})'.format(java_main, result))
+
+    self._rearrange_output_for_package(target_workdir, java_package)
+    if compiler == 'antlr3':
+      self._scrub_generated_timestamps(target_workdir)
+
+  def synthetic_target_extra_dependencies(self, target, target_workdir):
+    # Fetch the right java dependency from the target's compiler option
+    compiler_classpath_spec = self.get_options()[target.compiler]
+    return self.resolve_deps([compiler_classpath_spec])
+
+  # This checks to make sure that all of the sources have an identical package source structure, and
+  # if they do, uses that as the package. If they are different, then the user will need to set the
+  # package as it cannot be correctly inferred.
+  def _get_sources_package(self, target):
+    parents = set([os.path.dirname(source) for source in target.sources_relative_to_source_root()])
+    if len(parents) != 1:
+      raise self.AmbiguousPackageError('Antlr sources in multiple directories, cannot infer '
+                                       'package. Please set package member in antlr target.')
+    return parents.pop().replace('/', '.')
+
+  def _calculate_sources(self, targets):
+    sources = set()
+
+    def collect_sources(target):
+      if self.is_gentarget(target):
+        sources.update(target.sources_relative_to_buildroot())
+    for target in targets:
+      target.walk(collect_sources)
+    return sources
+
+  _COMMENT_WITH_TIMESTAMP_RE = re.compile('^//.*\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d')
+
+  def _rearrange_output_for_package(self, target_workdir, java_package):
+    """Rearrange the output files to match a standard Java structure.
+
+    Antlr emits a directory structure based on the relative path provided
+    for the grammar file. If the source root of the file is different from
+    the Pants build root, then the Java files end up with undesired parent
+    directories.
+    """
+    package_dir_rel = java_package.replace('.', os.path.sep)
+    package_dir = os.path.join(target_workdir, package_dir_rel)
+    safe_mkdir(package_dir)
+    for root, dirs, files in safe_walk(target_workdir):
+      if root == package_dir_rel:
+        # This path is already in the correct location
+        continue
+      for f in files:
+        os.rename(
+          os.path.join(root, f),
+          os.path.join(package_dir, f)
+        )
+
+    # Remove any empty directories that were left behind
+    for root, dirs, files in safe_walk(target_workdir, topdown = False):
+      for d in dirs:
+        full_dir = os.path.join(root, d)
+        if not os.listdir(full_dir):
+          os.rmdir(full_dir)
+
+  def _scrub_generated_timestamps(self, target_workdir):
+    """Remove the first line of comment from each file if it contains a timestamp."""
+    for root, _, filenames in safe_walk(target_workdir):
+      for filename in filenames:
+        source = os.path.join(root, filename)
+
+        with open(source) as f:
+          lines = f.readlines()
+        if len(lines) < 1:
+          return
+        with open(source, 'w') as f:
+          if not self._COMMENT_WITH_TIMESTAMP_RE.match(lines[0]):
+            f.write(lines[0])
+          for line in lines[1:]:
+            f.write(line)

src/python/pants/backend/codegen/antlr/java/java_antlr_library.py

@@ -0,0 +1,46 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.jvm.targets.jvm_target import JvmTarget
+
+
+class JavaAntlrLibrary(JvmTarget):
+  """A Java library generated from Antlr grammar files."""
+
+  def __init__(self,
+               name=None,
+               sources=None,
+               provides=None,
+               excludes=None,
+               compiler='antlr3',
+               package=None,
+               **kwargs):
+
+    """
+    :param provides: The ``artifact``
+      to publish that represents this target outside the repo.
+    :param compiler: The name of the compiler used to compile the ANTLR files.
+        Currently only supports 'antlr3' and 'antlr4'
+    :param package: (antlr4 only) A string which specifies the package to be used on the dependent
+        sources.  If unset, the package will be based on the path to the sources. Note that if
+        the sources are spread among different files, this must be set as the package cannot be
+        inferred.
+    """
+
+    super(JavaAntlrLibrary, self).__init__(name=name,
+                                           sources=sources,
+                                           provides=provides,
+                                           excludes=excludes,
+                                           **kwargs)
+    if not sources:
+      raise ValueError("Missing required 'sources' parameter.")
+    self.sources = sources
+
+    if compiler not in ('antlr3', 'antlr4'):
+      raise ValueError("Illegal value for 'compiler': {}".format(compiler))
+    self.compiler = compiler
+    self.package = package

src/python/pants/backend/codegen/antlr/java/register.py

@@ -0,0 +1,23 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.codegen.antlr.java.antlr_gen import AntlrGen
+from pants.backend.codegen.antlr.java.java_antlr_library import JavaAntlrLibrary
+from pants.build_graph.build_file_aliases import BuildFileAliases
+from pants.goal.task_registrar import TaskRegistrar as task
+
+
+def build_file_aliases():
+  return BuildFileAliases(
+    targets={
+      'java_antlr_library': JavaAntlrLibrary,
+    }
+  )
+
+
+def register_goals():
+  task(name='antlr', action=AntlrGen).install('gen')

src/python/pants/backend/codegen/antlr/python/BUILD

@@ -0,0 +1,8 @@
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies = [
+    'src/python/pants/backend/python/targets:python',
+  ],
+)

src/python/pants/backend/codegen/antlr/python/python_antlr_library.py

@@ -0,0 +1,23 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.python.targets.python_target import PythonTarget
+
+
+class PythonAntlrLibrary(PythonTarget):
+  """A Python library generated from Antlr grammar files."""
+
+  def __init__(self, module=None, antlr_version='3.1.3', *args, **kwargs):
+    """
+    :param module: everything beneath module is relative to this module name, None if root namespace
+    :param antlr_version:
+    """
+
+    super(PythonAntlrLibrary, self).__init__(*args, **kwargs)
+
+    self.module = module
+    self.antlr_version = antlr_version

src/python/pants/backend/codegen/jaxb/BUILD

@@ -0,0 +1,13 @@
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies = [
+    'src/python/pants/backend/jvm/targets:java',
+    'src/python/pants/backend/jvm/tasks:nailgun_task',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/base:payload',
+    'src/python/pants/base:payload_field',
+    'src/python/pants/task',
+  ],
+)