Requirements on language-specific sources should be optional. (pantsb…

…uild#6375)

E.g., a repo may have java without scala or vice versa.

This only works today because we implicitly expect codegen tasks
(which produce java and scala), and the deferred sources mapper task, to
be registered.  But not having them registered is a totally
reasonable thing to do (e.g., it improves start up time because
there are fewer options to parse).

This extends the logic of pantsbuild#6357.

The goal is to be able to run pants with just the backends you want.
In fact, once this is possible we should remove all the default registrations
(keeping, say, just graph_info and project_info) and require you to
opt in to whatever lang-specific backends you need.

Debatably, many more, possibly even all, requirements should be optional.
But that's a bigger issue, for another day.

src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py

@@ -62,7 +62,7 @@ def register_options(cls, register):
   def prepare(cls, options, round_manager):
     super(ProtobufGen, cls).prepare(options, round_manager)
     round_manager.require_data(JarImportProducts)
-    round_manager.require_data('deferred_sources')
+    round_manager.optional_data('deferred_sources')
   # TODO https://github.com/pantsbuild/pants/issues/604 prep finish
 
   def __init__(self, *args, **kwargs):

src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py

@@ -160,11 +160,11 @@ def prepare(cls, options, round_manager):
     # predicate to filter the full build graph is exposed, the requirement can be made automatic
     # and in turn codegen tasks could denote the labels they produce automating wiring of the
     # produce side
-    round_manager.require_data('java')
-    round_manager.require_data('scala')
+    round_manager.optional_data('java')
+    round_manager.optional_data('scala')
 
     # Allow the deferred_sources_mapping to take place first
-    round_manager.require_data('deferred_sources')
+    round_manager.optional_data('deferred_sources')
 
   # Subclasses must implement.
   # --------------------------

src/python/pants/backend/jvm/tasks/jvm_dependency_usage.py

@@ -75,9 +75,9 @@ def prepare(cls, options, round_manager):
       round_manager.require_data('product_deps_by_src')
     else:
       # We want to have synthetic targets in build graph to deserialize nodes properly.
-      round_manager.require_data('java')
-      round_manager.require_data('scala')
-      round_manager.require_data('deferred_sources')
+      round_manager.optional_data('java')
+      round_manager.optional_data('scala')
+      round_manager.optional_data('deferred_sources')
 
   @classmethod
   def skip(cls, options):

src/python/pants/backend/project_info/tasks/export.py

@@ -120,8 +120,8 @@ def register_options(cls, register):
   def prepare(cls, options, round_manager):
     super(ExportTask, cls).prepare(options, round_manager)
     if options.libraries or options.libraries_sources or options.libraries_javadocs:
-      round_manager.require_data('java')
-      round_manager.require_data('scala')
+      round_manager.optional_data('java')
+      round_manager.optional_data('scala')
 
   @memoized_property
   def _interpreter_cache(self):

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -58,7 +58,7 @@ def product_types(cls):
   @classmethod
   def prepare(cls, options, round_manager):
     round_manager.require_data(PythonInterpreter)
-    round_manager.require(SharedLibrary)
+    round_manager.optional_product(SharedLibrary)
 
   @classmethod
   def implementation_version(cls):

tests/python/pants_test/backend/jvm/BUILD

@@ -9,3 +9,13 @@ python_tests(
     'src/python/pants/java/jar',
   ]
 )
+
+python_tests(
+  name='integration',
+  sources=globs('*_integration.py'),
+  dependencies=[
+    'tests/python/pants_test:int-test',
+  ],
+  tags={'integration'},
+  timeout=300
+)

tests/python/pants_test/backend/jvm/test_backend_independence_integration.py

@@ -0,0 +1,29 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class BackendIndependenceTest(PantsRunIntegrationTest):
+  """Verifies that this backend works with no other backends present."""
+
+  @classmethod
+  def hermetic(cls):
+    return True
+
+  def test_independent_test_run(self):
+    pants_run = self.run_pants(
+      command=['test', 'examples/tests/java/org/pantsbuild/example/hello/greet'],
+      config={
+        'GLOBAL': {
+          'pythonpath': ['%(buildroot)s/pants-plugins/src/python'],
+          # We need the internal backend for the repo specifications in the example BUILD files.
+          # TODO: Don't use the real pants repos in example code, which are ideally self-contained.
+          'backend_packages': ['pants.backend.jvm', 'internal_backend.repositories'],
+        }
+      }
+    )
+    self.assert_success(pants_run)

tests/python/pants_test/backend/python/BUILD

@@ -48,6 +48,7 @@ python_tests(
   dependencies=[
     ':pants_requirement_integration_test_base',
     'src/python/pants/base:build_environment',
+    'tests/python/pants_test:int-test',
   ],
   tags={'integration'},
 )

tests/python/pants_test/backend/python/test_backend_independence_integration.py

@@ -0,0 +1,27 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class BackendIndependenceTest(PantsRunIntegrationTest):
+  """Verifies that this backend works with no other backends present."""
+
+  @classmethod
+  def hermetic(cls):
+    return True
+
+  def test_independent_test_run(self):
+    pants_run = self.run_pants(
+      command=['test', 'examples/tests/python/example_test/hello/greet'],
+      config={
+        'GLOBAL': {
+          'pythonpath': [],
+          'backend_packages': ['pants.backend.python'],
+        }
+      }
+    )
+    self.assert_success(pants_run)