Test for rule graph issues with each distinct backend (pantsbuild#10519)

We want to make sure that no matter what combination of backend packages you have loaded, Pants will not error due to issues with the rule graph.

Closes pantsbuild#10430.

[ci skip-rust]
[ci skip-build-wheels]

build-support/bin/generate_docs.py

@@ -27,7 +27,8 @@ class ReferenceGenerator:
     ./pants \
       --backend-packages="-['internal_backend.rules_for_testing', 'internal_backend.utilities']" \
       --backend-packages="+['pants.backend.python.lint.bandit', \
-        'pants.backend.python.lint.pylint', 'pants.backend.codegen.protobuf.python']" \
+        'pants.backend.python.lint.pylint', 'pants.backend.codegen.protobuf.python', \
+        'pants.backend.awslambda.python']" \
       --no-verify-config help-all > /tmp/help_info
 
     to generate the data, and then:

pants.toml

@@ -5,7 +5,6 @@ print_exception_stacktrace = true
 pythonpath = ["%(buildroot)s/pants-plugins/src/python"]
 
 backend_packages.add = [
-  "pants.backend.awslambda.python",
   "pants.backend.python",
   "pants.backend.python.lint.black",
   "pants.backend.python.lint.docformatter",

src/python/pants/init/BUILD

@@ -25,3 +25,10 @@ target(
     'src/python/pants/core',
   ],
 )
+
+python_integration_tests(
+  name="integration",
+  uses_pants_run=True,
+  # TODO(#10504): Lower this to ~120 once fixed.
+  timeout=450,
+)

src/python/pants/init/load_backends_integration_test.py

@@ -0,0 +1,41 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pathlib import Path
+from typing import List
+
+from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class LoadBackendsIntegrationTest(PantsRunIntegrationTest):
+    """Ensure that the rule graph can be loaded properly for each backend."""
+
+    @staticmethod
+    def discover_backends() -> List[str]:
+        register_pys = Path().glob("src/python/**/register.py")
+        backends = {
+            str(register_py.parent).replace("src/python/", "").replace("/", ".")
+            for register_py in register_pys
+        }
+        always_activated = {"pants.core", "pants.backend.project_info", "pants.backend.pants_info"}
+        return sorted(backends - always_activated)
+
+    def assert_backends_load(self, backends: List[str]) -> None:
+        result = self.run_pants(
+            ["--no-verify-config", "--version"], config={"GLOBAL": {"backend_packages": backends}}
+        )
+        self.assert_success(result, msg=f"Failed to load: {backends}")
+
+    def test_no_backends_loaded(self) -> None:
+        self.assert_backends_load([])
+
+    def test_all_backends_loaded(self) -> None:
+        """This should catch all ambiguity issues."""
+        all_backends = self.discover_backends()
+        self.assert_backends_load(all_backends)
+
+    def test_each_distinct_backend_loads(self) -> None:
+        """This should catch graph incompleteness errors, i.e. when a required rule is not
+        registered."""
+        for backend in self.discover_backends():
+            self.assert_backends_load([backend])