[internal] Replace pkgutil.get_data with new read_resource API (m…

…k 2) (pantsbuild#16485)

Pants uses `pkgutil.get_data()` in all manner of places to load resource files that are included within the Pants Python package. `get_data()` is not supported in the PEP302 pluggable importer used by PyOxidizer. [`importlib-resources`'s backport](https://pypi.org/project/importlib-resources/), however, _is_ supported.

`importlib-resources` (as far as PyOxidizer is concerned, at the very least) has some caveats:

* Resources can only be loaded from packages that contain an `__init__.py` file or namespace packages that do not contain `__init__.py`
* Resources cannot be `.py` files

This adds a shim function called `read_resource()` that allows reading resource files with more-or-less the same API as `pkgutil.get_data()` (in particular, allowing `read_resource(__name__, …)`, which is used commonly), but plugs into `importlib_resources.read_binary()` to allow for better portability.

Finally, the python dependency parser has been renamed to `dependency_parser_py`, so that it can be loaded as a resource, and still treated as a `python_source` for Pants linting purposes.

Addresses pantsbuild#7369.

[ci skip-build-wheels]

3rdparty/python/requirements.txt

@@ -13,6 +13,7 @@ freezegun==1.2.1
 # anonymity promise we make here: https://www.pantsbuild.org/docs/anonymous-telemetry
 humbug==0.2.7
 
+importlib_resources==5.0.*
 ijson==3.1.4
 packaging==21.3
 pex==2.1.103

src/python/pants/BUILD

@@ -13,7 +13,10 @@ python_test_utils(name="test_utils")
 
 python_distribution(
     name="pants-packaged",
-    dependencies=["./__main__.py", ":resources"],
+    dependencies=[
+        "./__main__.py",
+        ":resources",
+    ],
     # Because we have native code, this will cause the wheel to use whatever the ABI is for the
     # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`.
     sdist=False,

src/python/pants/backend/codegen/protobuf/scala/rules.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import os
-import pkgutil
 from dataclasses import dataclass
 
 from pants.backend.codegen import export_codegen_goal
@@ -55,6 +54,7 @@
 from pants.source.source_root import SourceRoot, SourceRootRequest
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet
+from pants.util.resources import read_resource
 
 
 class GenerateScalaFromProtobufRequest(GenerateSourcesRequest):
@@ -245,7 +245,7 @@ async def setup_scalapb_shim_classfiles(
 ) -> ScalaPBShimCompiledClassfiles:
     dest_dir = "classfiles"
 
-    scalapb_shim_content = pkgutil.get_data(
+    scalapb_shim_content = read_resource(
         "pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala"
     )
     if not scalapb_shim_content:

src/python/pants/backend/docker/subsystems/dockerfile_parser.py

@@ -4,7 +4,6 @@
 from __future__ import annotations
 
 import json
-import pkgutil
 from dataclasses import dataclass
 from pathlib import PurePath
 
@@ -35,6 +34,7 @@
 from pants.engine.unions import UnionRule
 from pants.util.docutil import git_url
 from pants.util.logging import LogLevel
+from pants.util.resources import read_resource
 
 _DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py"
 _DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems"
@@ -77,7 +77,7 @@ class ParserSetup:
 
 @rule
 async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:
-    parser_script_content = pkgutil.get_data(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)
+    parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)
     if not parser_script_content:
         raise ValueError(
             f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}."

src/python/pants/backend/go/go_sources/load_go_binary.py

@@ -3,7 +3,6 @@
 
 from __future__ import annotations
 
-import pkgutil
 from dataclasses import dataclass
 
 from pants.backend.go.util_rules.build_pkg import BuildGoPackageRequest, BuiltGoPackage
@@ -12,6 +11,7 @@
 from pants.engine.engine_aware import EngineAwareParameter
 from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.util.resources import read_resource
 
 
 @dataclass(frozen=True)
@@ -31,7 +31,7 @@ def debug_hint(self) -> str:
 
 def setup_files(dir_name: str, file_names: tuple[str, ...]) -> tuple[FileContent, ...]:
     def get_file(file_name: str) -> bytes:
-        content = pkgutil.get_data(f"pants.backend.go.go_sources.{dir_name}", file_name)
+        content = read_resource(f"pants.backend.go.go_sources.{dir_name}", file_name)
         if not content:
             raise AssertionError(f"Unable to find resource for `{file_name}`.")
         return content

src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py

@@ -4,7 +4,6 @@
 
 import json
 import os
-import pkgutil
 from dataclasses import dataclass
 from typing import Any, Iterator
 
@@ -25,6 +24,7 @@
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet
+from pants.util.resources import read_resource
 
 _PARSER_KOTLIN_VERSION = "1.6.20"
 
@@ -230,7 +230,7 @@ async def resolve_fallible_result_to_analysis(
 async def setup_kotlin_parser_classfiles(jdk: InternalJdk) -> KotlinParserCompiledClassfiles:
     dest_dir = "classfiles"
 
-    parser_source_content = pkgutil.get_data(
+    parser_source_content = read_resource(
         "pants.backend.kotlin.dependency_inference", "KotlinParser.kt"
     )
     if not parser_source_content:

src/python/pants/backend/python/dependency_inference/BUILD

@@ -1,7 +1,11 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_sources()
+python_sources(
+    dependencies=[
+        "./scripts:dependency_parser",
+    ],
+)
 
 python_tests(
     name="tests",

src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py

@@ -2,7 +2,6 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import json
-import pkgutil
 from dataclasses import dataclass
 
 from pants.backend.python.target_types import PythonSourceField
@@ -16,6 +15,7 @@
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
+from pants.util.resources import read_resource
 
 
 @dataclass(frozen=True)
@@ -61,7 +61,7 @@ class ParserScript:
 
 @rule
 async def parser_script() -> ParserScript:
-    script = pkgutil.get_data(__name__, "scripts/dependency_parser.py")
+    script = read_resource(__name__, "scripts/dependency_parser_py")
     assert script is not None
     return ParserScript(
         await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)]))

src/python/pants/backend/python/dependency_inference/scripts/BUILD

@@ -1,11 +1,15 @@
 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-resource(name="dependency_parser", source="dependency_parser.py")
+resource(
+    name="dependency_parser",
+    source="dependency_parser_py",
+)
+
 # Also expose scripts as python sources so they get formatted/linted/checked.
 python_source(
     name="dependency_parser_source",
-    source="dependency_parser.py",
+    source="dependency_parser_py",
     # This is run with Python 2.7 and 3.5+, so we shouldn't be running pyupgrade.
     # skip_pyupgrade=True,
 )

src/python/pants/backend/scala/dependency_inference/scala_parser.py

@@ -5,7 +5,6 @@
 import json
 import logging
 import os
-import pkgutil
 from dataclasses import dataclass
 from typing import Any, Iterator, Mapping
 
@@ -39,6 +38,7 @@
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet
+from pants.util.resources import read_resource
 
 logger = logging.getLogger(__name__)
 
@@ -316,7 +316,7 @@ async def resolve_fallible_result_to_analysis(
 async def setup_scala_parser_classfiles(jdk: InternalJdk) -> ScalaParserCompiledClassfiles:
     dest_dir = "classfiles"
 
-    parser_source_content = pkgutil.get_data(
+    parser_source_content = read_resource(
         "pants.backend.scala.dependency_inference", "ScalaParser.scala"
     )
     if not parser_source_content:

src/python/pants/backend/terraform/dependency_inference.py

@@ -2,7 +2,6 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 from __future__ import annotations
 
-import pkgutil
 from dataclasses import dataclass
 from pathlib import PurePath
 
@@ -36,6 +35,7 @@
 from pants.util.docutil import git_url
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import OrderedSet
+from pants.util.resources import read_resource
 
 
 class TerraformHcl2Parser(PythonToolRequirementsBase):
@@ -75,7 +75,7 @@ class ParserSetup:
 
 @rule
 async def setup_parser(hcl2_parser: TerraformHcl2Parser) -> ParserSetup:
-    parser_script_content = pkgutil.get_data("pants.backend.terraform", "hcl2_parser.py")
+    parser_script_content = read_resource("pants.backend.terraform", "hcl2_parser.py")
     if not parser_script_content:
         raise ValueError("Unable to find source to hcl2_parser.py wrapper script.")
 

src/python/pants/util/resources.py

@@ -0,0 +1,28 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+import importlib
+from itertools import chain
+
+import importlib_resources as resources
+
+
+def read_resource(package_or_module: str, resource: str) -> bytes:
+    """Reads a resource file from within the Pants package itself.
+
+    This helper function is designed for compatibility with `pkgutil.get_data()` wherever possible,
+    but also allows compability with PEP302 pluggable importers such as included with PyOxidizer.
+    """
+
+    a = importlib.import_module(package_or_module)
+    package_: str = a.__package__  # type: ignore[assignment]
+    resource_parts = resource.split("/")
+
+    if len(resource_parts) == 1:
+        package = package_
+    else:
+        package = ".".join(chain((package_,), resource_parts[:-1]))
+        resource = resource_parts[-1]
+
+    return resources.read_binary(package, resource)

src/python/pants/version.py

@@ -2,10 +2,12 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
-import pkgutil
 
 from packaging.version import Version
 
+# Generate a inferrable dependency on the `pants._version` package and its associated resources.
+from pants.util.resources import read_resource
+
 # Set this env var to override the version pants reports. Useful for testing.
 _PANTS_VERSION_OVERRIDE = "_PANTS_VERSION_OVERRIDE"
 
@@ -14,7 +16,7 @@
     os.environ.get(_PANTS_VERSION_OVERRIDE)
     or
     # NB: We expect VERSION to always have an entry and want a runtime failure if this is false.
-    pkgutil.get_data(__name__, "VERSION").decode().strip()  # type: ignore[union-attr]
+    read_resource(__name__, "VERSION").decode().strip()
 )
 
 PANTS_SEMVER = Version(VERSION)