Abstractions related to a JVM compiler's analysis.

Mostly refactorings of existing functionality.

Auditors: pl

(sapling split of bf1370b979f6d4f82c2ccb9e85104ca377e1d456)

src/python/twitter/pants/tasks/jvm_compile/__init__.py

@@ -0,0 +1 @@
+

src/python/twitter/pants/tasks/jvm_compile/analysis.py

@@ -0,0 +1,32 @@
+
+class Analysis(object):
+  """Parsed representation of an analysis for some JVM language.
+
+  An analysis provides information on the src -> class product mappings
+  and on the src -> {src|class|jar} file dependency mappings.
+  """
+  @classmethod
+  def merge(cls, analyses):
+    """Merge multiple analysis instances into one."""
+    raise NotImplementedError()
+
+  def split(self, splits, catchall=False):
+    """Split the analysis according to splits, which is a list of K iterables of source files.
+
+    If catchall is False, returns a list of K ZincAnalysis objects, one for each of the splits, in order.
+    If catchall is True, returns K+1 ZincAnalysis objects, the last one containing the analysis for any
+    remainder sources not mentioned in the K splits.
+    """
+    raise NotImplementedError()
+
+  def write_to_path(self, outfile_path, rebasings=None):
+    with open(outfile_path, 'w') as outfile:
+      self.write(outfile, rebasings)
+
+  def write(self, outfile, rebasings=None):
+    """Write this Analysis to outfile.
+
+    rebasings: A list of path prefix pairs [from_prefix, to_prefix] to rewrite.
+               to_prefix may be None, in which case matching paths are removed entirely.
+    """
+    raise NotImplementedError()

src/python/twitter/pants/tasks/jvm_compile/analysis_parser.py

@@ -0,0 +1,72 @@
+import os
+from twitter.pants import TaskError
+
+
+class ParseError(TaskError):
+  pass
+
+
+class AnalysisParser(object):
+  """Parse a file containing representation of an analysis for some JVM language."""
+
+  def is_nonempty_analysis(self, path):
+    """Returns whether an analysis at a specified path is nontrivial."""
+    if not os.path.exists(path):
+      return False
+    empty_prefix = self.empty_prefix()
+    with open(path, 'r') as infile:
+      prefix = infile.read(len(empty_prefix))
+    return prefix != empty_prefix
+
+  def empty_prefix(self):
+    """Returns a prefix indicating a trivial analysis file.
+
+    I.e., this prefix is present at the begnning of an analysis file iff the analysis is trivial.
+    """
+    raise NotImplementedError()
+
+  def parse_from_path(self, infile_path):
+    """Parse an analysis instance from a text file."""
+    with open(infile_path, 'r') as infile:
+      return self.parse(infile)
+
+  def parse(self, infile):
+    """Parse an analysis instance from an open file."""
+    raise NotImplementedError()
+
+  def parse_products_from_path(self, infile_path):
+    """An efficient parser of just the src->class mappings."""
+    with open(infile_path, 'r') as infile:
+      return self.parse_products(infile)
+
+  def parse_products(self, infile):
+    """An efficient parser of just the src->class mappings.
+
+    Returns a dict of src -> list of classfiles.
+    """
+    raise NotImplementedError()
+
+  def parse_deps_from_path(self, infile_path, classpath_indexer):
+    """An efficient parser of just the src->dep mappings.
+
+    classpath_indexer - a no-arg method that an implementation may call if it needs a mapping
+                        of class->element on the classpath that provides that class.
+                        We use this indirection to avoid unnecessary precomputation.
+    """
+    with open(infile_path, 'r') as infile:
+      return self.parse_deps(infile, classpath_indexer)
+
+  def parse_deps(self, infile, classpath_indexer):
+    """An efficient parser of just the binary, source and external deps sections.
+
+    classpath_indexer - a no-arg method that an implementation may call if it needs a mapping
+                        of class->element on the classpath that provides that class.
+                        We use this indirection to avoid unnecessary precomputation.
+
+    Returns a dict of src -> iterable of deps, where each item in deps is either a binary dep,
+    source dep or external dep, i.e., either a source file, a class file or a jar file.
+
+    All paths are absolute.
+    """
+    raise NotImplementedError()
+

src/python/twitter/pants/tasks/jvm_compile/analysis_tools.py

@@ -0,0 +1,82 @@
+import os
+import shutil
+from twitter.common.contextutil import temporary_dir
+from twitter.pants import get_buildroot
+
+
+class AnalysisTools(object):
+  """Analysis manipulation methods required by JvmCompile."""
+  _IVY_HOME_PLACEHOLDER = '/_IVY_HOME_PLACEHOLDER'
+  _PANTS_HOME_PLACEHOLDER = '/_PANTS_HOME_PLACEHOLDER'
+
+  def __init__(self, context, parser, analysis_cls):
+    self.parser = parser
+    self._java_home = context.java_home
+    self._ivy_home = context.ivy_home
+    self._pants_home = get_buildroot()
+    self._analysis_cls = analysis_cls
+
+  def split_to_paths(self, analysis_path, split_path_pairs, catchall_path=None):
+    """Split an analysis file.
+
+    split_path_pairs: A list of pairs (split, output_path) where split is a list of source files
+    whose analysis is to be split out into output_path. The source files may either be
+    absolute paths, or relative to the build root.
+
+    If catchall_path is specified, the analysis for any sources not mentioned in the splits is
+    split out to that path.
+    """
+    analysis = self.parser.parse_from_path(analysis_path)
+    splits = [x[0] for x in split_path_pairs]
+    split_analyses = analysis.split(splits, catchall_path is not None)
+    output_paths = [x[1] for x in split_path_pairs]
+    if catchall_path is not None:
+      output_paths.append(catchall_path)
+    for analysis, path in zip(split_analyses, output_paths):
+      analysis.write_to_path(path)
+
+  def merge_from_paths(self, analysis_paths, merged_analysis_path):
+    """Merge multiple analysis files into one."""
+    analyses = [self.parser.parse_from_path(path) for path in analysis_paths]
+    merged_analysis = self._analysis_cls.merge(analyses)
+    merged_analysis.write_to_path(merged_analysis_path)
+
+  def relativize(self, src_analysis, relativized_analysis):
+    with temporary_dir() as tmp_analysis_dir:
+      tmp_analysis_file = os.path.join(tmp_analysis_dir, 'analysis.relativized')
+
+      # NOTE: We can't port references to deps on the Java home. This is because different JVM
+      # implementations on different systems have different structures, and there's not
+      # necessarily a 1-1 mapping between Java jars on different systems. Instead we simply
+      # drop those references from the analysis file.
+      #
+      # In practice the JVM changes rarely, and it should be fine to require a full rebuild
+      # in those rare cases.
+      rebasings = [
+        (self._java_home, None),
+        (self._ivy_home, self._IVY_HOME_PLACEHOLDER),
+        (self._pants_home, self._PANTS_HOME_PLACEHOLDER),
+        ]
+      # Work on a tmpfile, for safety.
+      self._rebase_from_path(src_analysis, tmp_analysis_file, rebasings)
+      shutil.move(tmp_analysis_file, relativized_analysis)
+
+  def localize(self, src_analysis, localized_analysis):
+    with temporary_dir() as tmp_analysis_dir:
+      tmp_analysis_file = os.path.join(tmp_analysis_dir, 'analysis')
+      rebasings = [
+        (AnalysisTools._IVY_HOME_PLACEHOLDER, self._ivy_home),
+        (AnalysisTools._PANTS_HOME_PLACEHOLDER, self._pants_home),
+        ]
+      # Work on a tmpfile, for safety.
+      self._rebase_from_path(src_analysis, tmp_analysis_file, rebasings)
+      shutil.move(tmp_analysis_file, localized_analysis)
+
+  def _rebase_from_path(self, input_analysis_path, output_analysis_path, rebasings):
+    """Rebase file paths in an analysis file.
+
+    rebasings: A list of path prefix pairs [from_prefix, to_prefix] to rewrite.
+               to_prefix may be None, in which case matching paths are removed entirely.
+    """
+    analysis = self.parser.parse_from_path(input_analysis_path)
+    analysis.write_to_path(output_analysis_path, rebasings=rebasings)