Add a whitelist to jvm dependency analyzer

The whitelist allows big repos to reduce dependency issues while stopping any new from being introduced. You can whitelist the projects that currently have issues and make the dependency checks fatal. All projects with current issues will still be able to build but new issues will result in breaking builds.

Also it seems like the direct dependency check would always fatal if set to fatal, fixed that.

Testing Done:
Added two integration tests.

Reviewed at https://rbcommons.com/s/twitter/r/888/

src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py

@@ -19,6 +19,7 @@
 from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
 from pants.backend.jvm.tasks.nailgun_task import NailgunTaskBase
 from pants.base.build_environment import get_buildroot, get_scm
+from pants.base.config import Config
 from pants.base.exceptions import TaskError
 from pants.base.target import Target
 from pants.base.worker_pool import Work
@@ -215,11 +216,14 @@ def munge_flag(flag):
     check_unnecessary_deps = munge_flag(self._get_lang_specific_option('unnecessary_deps'))
 
     if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
+      target_whitelist = self.context.config.getlist('jvm', 'missing_deps_target_whitelist', default=[])
+
       # Must init it here, so it can set requirements on the context.
       self._dep_analyzer = JvmDependencyAnalyzer(self.context,
                                                  check_missing_deps,
                                                  check_missing_direct_deps,
-                                                 check_unnecessary_deps)
+                                                 check_unnecessary_deps,
+                                                 target_whitelist)
     else:
       self._dep_analyzer = None
 

src/python/pants/backend/jvm/tasks/jvm_compile/jvm_dependency_analyzer.py

@@ -24,7 +24,8 @@ def __init__(self,
                context,
                check_missing_deps,
                check_missing_direct_deps,
-               check_unnecessary_deps):
+               check_unnecessary_deps,
+               target_whitelist):
 
     self._context = context
     self._context.products.require_data('classes_by_target')
@@ -34,6 +35,9 @@ def __init__(self,
     self._check_missing_direct_deps = check_missing_direct_deps
     self._check_unnecessary_deps = check_unnecessary_deps
 
+    # These targets we will not report as having any dependency issues even if they do.
+    self._target_whitelist = OrderedSet(target_whitelist)
+
   def _compute_targets_by_file(self):
     """Returns a map from abs path of source, class or jar file to an OrderedSet of targets.
 
@@ -155,6 +159,13 @@ def shorten(path):  # Make the output easier to read.
             return os.path.relpath(path, prefix)
         return path
 
+      def filter_whitelisted(missing_deps):
+        # Removing any targets that exist in the whitelist from the list of dependency issues.
+        return [(tgt_pair, evidence) for (tgt_pair, evidence) in missing_deps
+                            if tgt_pair[0].address.reference() not in self._target_whitelist]
+
+      missing_tgt_deps = filter_whitelisted(missing_tgt_deps)
+
       if self._check_missing_deps and (missing_file_deps or missing_tgt_deps):
         for (tgt_pair, evidence) in missing_tgt_deps:
           evidence_str = '\n'.join(['    %s uses %s' % (shorten(e[0]), shorten(e[1]))
@@ -168,7 +179,11 @@ def shorten(path):  # Make the output easier to read.
         if self._check_missing_deps == 'fatal':
           raise TaskError('Missing deps.')
 
-      if self._check_missing_direct_deps:
+
+      missing_direct_tgt_deps = filter_whitelisted(missing_direct_tgt_deps)
+
+      if self._check_missing_direct_deps and missing_direct_tgt_deps:
+
         for (tgt_pair, evidence) in missing_direct_tgt_deps:
           evidence_str = '\n'.join(['    %s uses %s' % (shorten(e[0]), shorten(e[1]))
                                     for e in evidence])

testprojects/src/java/com/pants/testproject/missingdepswhitelist/BUILD

@@ -0,0 +1,10 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+java_library(name='missingdepswhitelist',
+  sources=rglobs('*.java'),
+  dependencies=[
+    'examples/src/java/com/pants/examples/hello/greet',
+    'testprojects/src/java/com/pants/testproject/missingdepswhitelist2'
+  ]
+)

testprojects/src/java/com/pants/testproject/missingdepswhitelist/MissingDepsWhitelist.java

@@ -0,0 +1,11 @@
+package com.pants.testproject.missingdepswhitelist;
+
+import com.pants.examples.hello.greet.Greeting;
+import com.pants.testproject.missingdepswhitelist2.MissingDepsWhitelist2;
+
+public class MissingDepsWhitelist {
+  public String doStuff() {
+    MissingDepsWhitelist2 scala = new MissingDepsWhitelist2();
+    return Greeting.greet("woop");
+  }
+}

testprojects/src/java/com/pants/testproject/missingdepswhitelist/README.md

@@ -0,0 +1,6 @@
+The JVM dependency analyzer will tell users if a target a is using classes it does not correctly depend on.
+
+However, in some cases (such as catching up on fixing these dependency issues) it is useful to have a
+whitelist of targets to not report or fail builds on.
+
+This project will generate a missing dep warning, but it is whitelisted so in the end it will not.

testprojects/src/java/com/pants/testproject/missingdepswhitelist2/BUILD

@@ -0,0 +1,6 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+java_library(name='missingdepswhitelist2',
+  sources=rglobs('*.java'),
+)

testprojects/src/java/com/pants/testproject/missingdepswhitelist2/MissingDepsWhitelist2.java

@@ -0,0 +1,9 @@
+package com.pants.testproject.missingdepswhitelist2;
+
+import com.pants.examples.hello.greet.Greeting;
+
+public class MissingDepsWhitelist2 {
+  public String doStuff() {
+    return Greeting.greet("weep");
+  }
+}

testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist/BUILD

@@ -0,0 +1,9 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+java_library(name='missingdirectdepswhitelist',
+  sources=rglobs('*.java'),
+  dependencies=[
+    'testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist2'
+  ]
+)

testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist/MissingDirectDepsWhitelist.java

@@ -0,0 +1,11 @@
+package com.pants.testproject.missingdirectdepswhitelist;
+
+import com.pants.examples.hello.greet.Greeting;
+import com.pants.testproject.missingdirectdepswhitelist2.MissingDirectDepsWhitelist2;
+
+public class MissingDirectDepsWhitelist {
+  public String doStuff() {
+    MissingDirectDepsWhitelist2 scala = new MissingDirectDepsWhitelist2();
+    return Greeting.greet("woop");
+  }
+}

testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist2/BUILD

@@ -0,0 +1,7 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+java_library(name='missingdirectdepswhitelist2',
+  sources=rglobs('*.java'),
+  dependencies=['examples/src/java/com/pants/examples/hello/greet']
+)

testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist2/MissingDirectDepsWhitelist2.java

@@ -0,0 +1,9 @@
+package com.pants.testproject.missingdirectdepswhitelist2;
+
+import com.pants.examples.hello.greet.Greeting;
+
+public class MissingDirectDepsWhitelist2 {
+  public String doStuff() {
+    return Greeting.greet("weep");
+  }
+}

tests/python/pants_test/tasks/jvm_compile/java/test_java_compile_integration.py

@@ -72,3 +72,44 @@ def test_java_compile_produces_different_artifact_depending_on_java_version(self
 
       # One artifact for java 6 and one for 7
       self.assertEqual(len(os.listdir(artifact_dir)), 2)
+
+
+  def _whitelist_test(self, target, fatal_flag, whitelist):
+    # We want to ensure that a project missing dependencies can be
+    # whitelisted so that the missing deps do not break the build.
+
+    args = [
+      'goal',
+      'compile',
+      target,
+      fatal_flag
+    ]
+
+    # First check that without the whitelist we do break the build.
+    pants_run = self.run_pants(args, {})
+    self.assertNotEquals(pants_run.returncode, self.PANTS_SUCCESS_CODE)
+
+    # Now let's use the target whitelist, this should succeed.
+    config = {
+      'jvm': {'missing_deps_target_whitelist': [whitelist]}
+    }
+
+    pants_run = self.run_pants(args, config)
+
+    self._assert_run_success(pants_run)
+
+
+  def test_java_compile_missing_dep_analysis_whitelist(self):
+    self._whitelist_test(
+      'testprojects/src/java/com/pants/testproject/missingdepswhitelist',
+      '--compile-java-missing-deps=fatal',
+      'testprojects/src/java/com/pants/testproject/missingdepswhitelist2'
+    )
+
+
+  def test_java_compile_missing_direct_dep_analysis_whitelist(self):
+    self._whitelist_test(
+      'testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist',
+      '--compile-java-missing-direct-deps=fatal',
+      'testprojects/src/java/com/pants/testproject/missingdirectdepswhitelist'
+    )