[models] Add sharding to all-files parse_test (RobotLocomotion#22344)

bindings/pydrake/common/test_utilities/BUILD.bazel

@@ -12,6 +12,7 @@ load(
 
 package(default_visibility = [
     "//bindings/pydrake:__subpackages__",
+    "//tools/workspace/drake_models:__pkg__",
 ])
 
 # This determines how `PYTHONPATH` is configured.
@@ -31,10 +32,6 @@ drake_py_library(
     name = "algebra_test_util_py",
     testonly = True,
     srcs = ["algebra_test_util.py"],
-    visibility = [
-        "//bindings/pydrake/autodiffutils:__pkg__",
-        "//bindings/pydrake/symbolic:__pkg__",
-    ],
     deps = [
         ":module_py",
     ],

tools/workspace/drake_models/BUILD.bazel

@@ -9,13 +9,15 @@ enumerate_filegroup(
 
 drake_py_unittest(
     name = "parse_test",
-    timeout = "long",
+    timeout = "moderate",
     data = [
         ":inventory.txt",
         "@drake_models",
     ],
+    shard_count = 16,
     deps = [
         "//bindings/pydrake",
+        "//bindings/pydrake/common/test_utilities:meta_py",
         "@rules_python//python/runfiles",
     ],
 )

tools/workspace/drake_models/test/parse_test.py

@@ -4,97 +4,105 @@
 
 from python import runfiles
 
+from pydrake.common.test_utilities.meta import (
+    ValueParameterizedTest,
+    run_with_multiple_values,
+)
 from pydrake.planning import RobotDiagramBuilder
 
 
-class TestDrakeModels(unittest.TestCase):
+def _runfiles_inventory() -> Iterator[tuple[str, Path]]:
+    """Generates (resource_path, filesystem_path) tuples for all runfiles in
+    @drake_models. For example, resource_path="drake_models/veggies/pepper.png"
+    and filesystem_path="/dir/to/foo.runfiles/drake_models/veggies/pepper.png".
+    """
+    manifest = runfiles.Create()
+    inventory = Path(manifest.Rlocation(
+        "drake/tools/workspace/drake_models/inventory.txt"))
+    repo_name = "drake_models/"
+    for line in inventory.read_text(encoding="utf-8").splitlines():
+        assert line.startswith(repo_name), line
+        filename = line[len(repo_name):].strip()
+        resource_path = f"drake_models/{filename}"
+        filesystem_path = Path(manifest.Rlocation(resource_path))
+        assert filesystem_path.exists(), filesystem_path
+        yield resource_path, filesystem_path
 
-    @staticmethod
-    def package_xml() -> Path:
-        """Returns the path to drake_models/package.xml in runfiles.
-        """
-        manifest = runfiles.Create()
-        result = Path(manifest.Rlocation("drake_models/package.xml"))
-        assert result.exists(), result
-        return result
-
-    @staticmethod
-    def inventory() -> Iterator[str]:
-        """Generates the list of all the runfiles of @drake_models. The result
-        is strings like "drake_models/veggies/pepper.png".
-        """
-        manifest = runfiles.Create()
-        inventory = Path(manifest.Rlocation(
-            "drake/tools/workspace/drake_models/inventory.txt"))
-        with open(inventory, encoding="utf-8") as f:
-            for line in f.readlines():
-                assert line.startswith("drake_models/")
-                yield line.strip()
 
-    @staticmethod
-    def get_all_models() -> Iterator[str]:
-        """Generates the Paths of all top-level model files in @drake_models.
-        The result is URLs like "package://drake_models/veggies/pepper.sdf".
-        """
-        for path in TestDrakeModels.inventory():
-            url = f"package://{path}"
-            if any([path.endswith(".dmd.yaml"),
-                    path.endswith(".sdf"),
-                    path.endswith(".urdf")]):
-                yield url
-            elif path.endswith(".xml"):
-                # We can't tell a MuJoCo file just by its suffix.
-                runfiles = TestDrakeModels.package_xml().parent.parent
-                if "<mujoco" in (runfiles / path).open().read():
+def _models_inventory() -> Iterator[str]:
+    """Generates package URLs for all top-level model files in @drake_models.
+    For example, url="package://drake_models/veggies/pepper.sdf".
+    """
+    for resource_path, filesystem_path in _runfiles_inventory():
+        url = f"package://{resource_path}"
+        if any([resource_path.endswith(".dmd.yaml"),
+                resource_path.endswith(".sdf"),
+                resource_path.endswith(".urdf")]):
+            yield url
+        elif resource_path.endswith(".xml"):
+            # We can't tell a MuJoCo file just by its suffix.
+            with filesystem_path.open(encoding="utf-8") as xml:
+                if "<mujoco" in xml.read(1024):
                     yield url
-        return
 
-    def test_all_models(self):
-        """Checks that all model files in @drake_models parse without any
-        errors nor warnings.
-        """
-        all_models = list(TestDrakeModels.get_all_models())
 
-        # Allow warnings on these models until they are repaired.
-        models_with_warnings = [
-            # TODO(#19992) for tracking these warnings.
-            "package://drake_models/atlas/atlas_convex_hull.urdf",
-            "package://drake_models/atlas/atlas_minimal_contact.urdf",
-            # We don't have any tracking issue for fixing the allegro models,
-            # because we don't use them for anything we care about. If someone
-            # wants to fix the warnings, be our guest.
-            "package://drake_models/allegro_hand_description/urdf/allegro_hand_description_left.urdf",  # noqa
-            "package://drake_models/allegro_hand_description/urdf/allegro_hand_description_right.urdf",  # noqa
-            # TODO(jwnimmer-tri) Fix these warnings.
-            "package://drake_models/jaco_description/urdf/j2n6s300_col.urdf",
-        ]
-        self.assertFalse(set(models_with_warnings) - set(all_models))
+def _get_all_test_cases() -> Iterator[dict]:
+    """Generates test case specifications, each one a dictionary consisting of:
+    - url: str
+    - expect_warnings: bool
+    - expect_errors: bool
+    The test specifications are is used by TestDrakeModels.test_model below to
+    run Drake's parser against all models in @drake_models.
+    """
+    # Package URLs for all top-level model files.
+    all_models = set(_models_inventory())
+
+    # Allow warnings on these models until they are repaired.
+    models_with_warnings = set([
+        # TODO(#19992) for tracking these warnings.
+        "package://drake_models/atlas/atlas_convex_hull.urdf",
+        "package://drake_models/atlas/atlas_minimal_contact.urdf",
+        # We don't have any tracking issue for fixing the allegro models,
+        # because we don't use them for anything we care about. If someone
+        # wants to fix the warnings, be our guest.
+        "package://drake_models/allegro_hand_description/urdf/allegro_hand_description_left.urdf",  # noqa
+        "package://drake_models/allegro_hand_description/urdf/allegro_hand_description_right.urdf",  # noqa
+        # TODO(jwnimmer-tri) Fix these warnings.
+        "package://drake_models/jaco_description/urdf/j2n6s300_col.urdf",
+    ])
+    stragglers = models_with_warnings - all_models
+    assert not stragglers, repr(stragglers)
+
+    # Allow errors on these models until they are repaired.
+    models_with_errors = set([
+        # This file is not designed to be loaded independently from its
+        # parent file `homecart.dmd.yaml`; it will always error out.
+        "package://drake_models/tri_homecart/homecart_grippers.dmd.yaml",
+    ])
+    stragglers = models_with_errors - all_models
+    assert not stragglers, repr(stragglers)
+    overlap = models_with_warnings & models_with_errors
+    assert not overlap, repr(overlap)
+
+    for url in all_models:
+        yield dict(
+            url=url,
+            expect_warnings=(url in models_with_warnings),
+            expect_errors=(url in models_with_errors),
+        )
 
-        # Allow errors on these models until they are repaired.
-        models_with_errors = [
-            # This file is not designed to be loaded independently from its
-            # parent file `homecart.dmd.yaml`; it will always error out.
-            "package://drake_models/tri_homecart/homecart_grippers.dmd.yaml",
-        ]
-        self.assertFalse(set(models_with_errors) - set(all_models))
-        self.assertFalse(set(models_with_warnings) & set(models_with_errors))
 
-        # Run all tests.
-        for url in all_models:
-            with self.subTest(url=url):
-                expect_warnings = url in models_with_warnings
-                expect_errors = url in models_with_errors
-                self._check_model(
-                    url=url,
-                    expect_warnings=expect_warnings,
-                    expect_errors=expect_errors)
+class TestDrakeModels(unittest.TestCase, metaclass=ValueParameterizedTest):
 
-    def _check_model(self, *,
-                     url: str,
-                     expect_warnings: bool,
-                     expect_errors: bool):
-        """Checks that when parsed, the given url contains warnings and/or
-        errors consistent with the expected conditions.
+    @run_with_multiple_values(_get_all_test_cases())
+    def test_model(self, *,
+                   url: str,
+                   expect_warnings: bool,
+                   expect_errors: bool):
+        """Checks that when parsed, the given package url contains warnings
+        and/or errors consistent with the given conditions. This test case is
+        run repeatedly (via 'run_with_multiple_values') in order to cover all
+        models in @drake_models.
         """
         if expect_errors:
             with self.assertRaises(Exception):
@@ -110,7 +118,6 @@ def _check_model(self, *,
     def _parse(self, *, url: str, strict: bool):
         builder = RobotDiagramBuilder()
         parser = builder.parser()
-        parser.package_map().AddPackageXml(str(self.package_xml()))
         if strict:
             parser.SetStrictParsing()
         parser.AddModels(url=url)