From 37bf6071b9dea6074176cbc67e97cdb70fccb71d Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Mon, 15 Aug 2022 11:37:49 -0700 Subject: [PATCH] [internal] Replace `pkgutil.get_data` with new `read_resource` API (mk 2) (#16485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pants uses `pkgutil.get_data()` in all manner of places to load resource files that are included within the Pants Python package. `get_data()` is not supported in the PEP302 pluggable importer used by PyOxidizer. [`importlib-resources`'s backport](https://pypi.org/project/importlib-resources/), however, _is_ supported. `importlib-resources` (as far as PyOxidizer is concerned, at the very least) has some caveats: * Resources can only be loaded from packages that contain an `__init__.py` file or namespace packages that do not contain `__init__.py` * Resources cannot be `.py` files This adds a shim function called `read_resource()` that allows reading resource files with more-or-less the same API as `pkgutil.get_data()` (in particular, allowing `read_resource(__name__, …)`, which is used commonly), but plugs into `importlib_resources.read_binary()` to allow for better portability. Finally, the python dependency parser has been renamed to `dependency_parser_py`, so that it can be loaded as a resource, and still treated as a `python_source` for Pants linting purposes. Addresses #7369. [ci skip-build-wheels] --- 3rdparty/python/requirements.txt | 1 + 3rdparty/python/user_reqs.lock | 125 +++++++++++------- src/python/pants/BUILD | 5 +- .../backend/codegen/protobuf/scala/rules.py | 4 +- .../docker/subsystems/dockerfile_parser.py | 4 +- .../backend/go/go_sources/load_go_binary.py | 4 +- .../dependency_inference/kotlin_parser.py | 4 +- .../backend/python/dependency_inference/BUILD | 6 +- .../parse_python_dependencies.py | 4 +- .../python/dependency_inference/scripts/BUILD | 8 +- ...endency_parser.py => dependency_parser_py} | 0 .../dependency_inference/scala_parser.py | 4 +- .../backend/terraform/dependency_inference.py | 4 +- src/python/pants/util/resources.py | 28 ++++ src/python/pants/version.py | 6 +- 15 files changed, 141 insertions(+), 66 deletions(-) rename src/python/pants/backend/python/dependency_inference/scripts/{dependency_parser.py => dependency_parser_py} (100%) create mode 100644 src/python/pants/util/resources.py diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 5307da630d7..ee05e23c3d3 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -13,6 +13,7 @@ freezegun==1.2.1 # anonymity promise we make here: https://www.pantsbuild.org/docs/anonymous-telemetry humbug==0.2.7 +importlib_resources==5.0.* ijson==3.1.4 packaging==21.3 pex==2.1.103 diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index 5fd5e95cefc..47f25a96991 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -4,7 +4,7 @@ // // --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- // { -// "version": 2, +// "version": 3, // "valid_for_interpreter_constraints": [ // "CPython<3.10,>=3.7" // ], @@ -19,6 +19,7 @@ // "freezegun==1.2.1", // "humbug==0.2.7", // "ijson==3.1.4", +// "importlib_resources==5.0.*", // "mypy-typing-asserts==0.1.1", // "packaging==21.3", // "pex==2.1.103", @@ -40,7 +41,8 @@ // "types-toml==0.10.8", // "typing-extensions==4.3.0", // "uvicorn[standard]==0.17.6" -// ] +// ], +// "requirement_constraints": [] // } // --- END PANTS LOCKFILE METADATA --- @@ -793,6 +795,36 @@ "requires_python": ">=3.7", "version": "4.12" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "2238159eb743bd85304a16e0536048b3e991c531d1cd51c4a834d1ccf2829057", + "url": "https://files.pythonhosted.org/packages/46/10/7cc167fe072037c3cd2a15a92bb963b86f2bab8ac0995fab95fb7a152b80/importlib_resources-5.0.7-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "4df460394562b4581bb4e4087ad9447bd433148fba44241754ec3152499f1d1b", + "url": "https://files.pythonhosted.org/packages/4a/d5/22aa0454c06788e59f406a2b0e569fac835c6c45e5ad6ed968804920f0ac/importlib_resources-5.0.7.tar.gz" + } + ], + "project_name": "importlib-resources", + "requires_dists": [ + "jaraco.packaging>=8.2; extra == \"docs\"", + "pytest!=3.7.3,>=3.5; extra == \"testing\"", + "pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-checkdocs>=1.2.3; extra == \"testing\"", + "pytest-cov; extra == \"testing\"", + "pytest-enabler; extra == \"testing\"", + "pytest-flake8; extra == \"testing\"", + "pytest-mypy; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "rst.linker>=1.9; extra == \"docs\"", + "sphinx; extra == \"docs\"", + "zipp>=0.4; python_version < \"3.8\"" + ], + "requires_python": ">=3.6", + "version": "5.0.7" + }, { "artifacts": [ { @@ -980,98 +1012,98 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "4988c0f13c42bfa9ddd2fe2f569c9d54646ce84adc5de84228cfe83396f3bd58", - "url": "https://files.pythonhosted.org/packages/f3/88/78666bfe38d3a8aee75fbd2410ac6e26dfdd64585323c07648f387817c76/pydantic-1.9.1-py3-none-any.whl" + "hash": "78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e", + "url": "https://files.pythonhosted.org/packages/fe/27/0de772dcd0517770b265dbc3998ed3ee3aa2ba25ba67e3685116cbbbccc6/pydantic-1.9.2-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "72ccb318bf0c9ab97fc04c10c37683d9eea952ed526707fabf9ac5ae59b701fd", - "url": "https://files.pythonhosted.org/packages/2e/6b/7ae3031fc86b974296ef3a91221aec46fdf66a8dd6ba1d300151a812c5b3/pydantic-1.9.1-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb", + "url": "https://files.pythonhosted.org/packages/04/bc/5231387df42b199f38dd3f29eb10338bc0a272e24020aff5c4cd64d3270d/pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "02eefd7087268b711a3ff4db528e9916ac9aa18616da7bca69c1871d0b7a091f", - "url": "https://files.pythonhosted.org/packages/5f/76/11635fe2d808c0062f4f23a6ac6453e6110d3446fb196e3b6dfc1d15f979/pydantic-1.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8", + "url": "https://files.pythonhosted.org/packages/18/2f/228fe5d1dbf7c36bd252fb304b015d02a50f696e659a0bb370a5628d00f4/pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "494f7c8537f0c02b740c229af4cb47c0d39840b829ecdcfc93d91dcbb0779892", - "url": "https://files.pythonhosted.org/packages/71/54/de851f0ab24e6044be1df7186ba000defefe36945f10b86f70f6b4300284/pydantic-1.9.1-cp37-cp37m-macosx_10_9_x86_64.whl" + "hash": "f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801", + "url": "https://files.pythonhosted.org/packages/6f/1e/4dca34af2a7e8effb5226ac2fec3664e99c8e95c97e8ebae9ff47fb3bbef/pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "9bcf8b6e011be08fb729d110f3e22e654a50f8a826b0575c7196616780683380", - "url": "https://files.pythonhosted.org/packages/80/30/553e846151cd7dbd2405fb128272c6cea5ea3e42f9538e2962838dcf755b/pydantic-1.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl" + "hash": "a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f", + "url": "https://files.pythonhosted.org/packages/75/44/e3c3c72ddbf7f6c987e39cc09f21f61f21cffeebddb75b9019f952624942/pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "4ce9ae9e91f46c344bec3b03d6ee9612802682c1551aaf627ad24045ce090761", - "url": "https://files.pythonhosted.org/packages/81/48/b3c8cb4eb7106b612bbfc263d557b75e27d2720a0cf2c442ea67950d6d43/pydantic-1.9.1-cp39-cp39-musllinux_1_1_i686.whl" + "hash": "e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa", + "url": "https://files.pythonhosted.org/packages/84/cf/b2514b857196fb8484209c6bf365a164b684f6eef3d1feaa4f9ce2447389/pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "a9af62e9b5b9bc67b2a195ebc2c2662fdf498a822d62f902bf27cccb52dbbf49", - "url": "https://files.pythonhosted.org/packages/8e/d8/56552e89a644b77e51fd8c2c6cc4982ec83b5f6221c10c00bf1406255fec/pydantic-1.9.1-cp38-cp38-musllinux_1_1_i686.whl" + "hash": "91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3", + "url": "https://files.pythonhosted.org/packages/86/f8/c2effc693180e16b3ec886bc9d080f937afa7964823a7c204d5c9df55264/pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "9ce157d979f742a915b75f792dbd6aa63b8eccaf46a1005ba03aa8a986bde34a", - "url": "https://files.pythonhosted.org/packages/97/7b/229158bcd271f02ec2e92c6740ce47236797fcef2d36cbe949c0ea9f3a68/pydantic-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl" + "hash": "e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747", + "url": "https://files.pythonhosted.org/packages/88/83/42a71762ec2f127ba8141a0608dea0ee2a8aa2dd6fcc0d2cda375aee61eb/pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "b83ba3825bc91dfa989d4eed76865e71aea3a6ca1388b59fc801ee04c4d8d0d6", - "url": "https://files.pythonhosted.org/packages/9a/a2/585b1a747b7bbb22392fe9c875b8adfaaee9a7be506fbb66a749157f9099/pydantic-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56", + "url": "https://files.pythonhosted.org/packages/95/5b/5d1d8d5e6e2d9a1ec3a94b75b14fe5a2e6efd13fa96a3e53144db9de9d48/pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "969dd06110cb780da01336b281f53e2e7eb3a482831df441fb65dd30403f4608", - "url": "https://files.pythonhosted.org/packages/b9/01/fad5d5a724780873e0fac2b1ca9365636179921fc371ac6a7ec6b9706d8e/pydantic-1.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4", + "url": "https://files.pythonhosted.org/packages/a0/cb/672a6e3a9fa78c9a21f274dbdef7f20633969527f07ac8f882263844f4c1/pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "0bf07cab5b279859c253d26a9194a8906e6f4a210063b84b433cf90a569de0c1", - "url": "https://files.pythonhosted.org/packages/c1/3b/e3f31676b694ab293ba4b7c5a3146fa8ec72c7a4d71c69a883aad1210517/pydantic-1.9.1-cp38-cp38-macosx_11_0_arm64.whl" + "hash": "1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44", + "url": "https://files.pythonhosted.org/packages/bb/9c/7ded003135342ea07fcac5581790634a2d70340175c1e7cb2f0affcb1962/pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "7eb57ba90929bac0b6cc2af2373893d80ac559adda6933e562dcfb375029acee", - "url": "https://files.pythonhosted.org/packages/c4/d2/6118efdb9fdaf3d4dfecc0276d4d47a1ef9aaf9903fa49f1ece765d917cb/pydantic-1.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0", + "url": "https://files.pythonhosted.org/packages/be/72/841dcb62c23d8955b82784dd3bb73770d1ce8aa562e5bd47c1f52230ca12/pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "fe4670cb32ea98ffbf5a1262f14c3e102cccd92b1869df3bb09538158ba90fe6", - "url": "https://files.pythonhosted.org/packages/c7/26/f5d5c0e9d3bca2c720a70b3280299dd76861b1938862cd4425d7d7e88396/pydantic-1.9.1-cp38-cp38-musllinux_1_1_x86_64.whl" + "hash": "4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8", + "url": "https://files.pythonhosted.org/packages/d3/4b/6f539c1f26c6a8ed942fa751981909ab86336ce5ead28b6c92590ee6bc1b/pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "1ed987c3ff29fff7fd8c3ea3a3ea877ad310aae2ef9889a119e22d3f2db0691a", - "url": "https://files.pythonhosted.org/packages/d0/a5/e4a25a0becf35530a3d90459a88855743e942f2e502da49ca5b10aa78568/pydantic-1.9.1.tar.gz" + "hash": "cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979", + "url": "https://files.pythonhosted.org/packages/e0/0f/a8adcc49e58994f6da6b96dac42dedbedd250c3130d59a664d8130c8019d/pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "5d93d4e95eacd313d2c765ebe40d49ca9dd2ed90e5b37d0d421c597af830c195", - "url": "https://files.pythonhosted.org/packages/de/0a/bd6eb16c07eba6aad5e543985995e2cbdc4925ccffc2f9491482f9101c0e/pydantic-1.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453", + "url": "https://files.pythonhosted.org/packages/e9/64/3395d45a05adcebb6d1025702c28d1ed188703397f38999295c52687f87e/pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "1542636a39c4892c4f4fa6270696902acb186a9aaeac6f6cf92ce6ae2e88564b", - "url": "https://files.pythonhosted.org/packages/df/c9/7ea0a065d96a200f008021f28149f16b6d99c30d04461c562061d6a73ce8/pydantic-1.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d", + "url": "https://files.pythonhosted.org/packages/ed/c9/ffe44727dadb0930783a1ffb60facf8ead7dffbb67db9ae2fa28dacabcf1/pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "177071dfc0df6248fd22b43036f936cfe2508077a72af0933d0c1fa269b18537", - "url": "https://files.pythonhosted.org/packages/ea/3a/6b451f0ddb374e6a23642554ee3a357029d718083e084b3c9e6ea9b8119a/pydantic-1.9.1-cp37-cp37m-musllinux_1_1_i686.whl" + "hash": "4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3", + "url": "https://files.pythonhosted.org/packages/f6/63/b412252dbbdc712500ad73fe2e591c3220781e63a8c135d26b7d60fcb99c/pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "1dd8fecbad028cd89d04a46688d2fcc14423e8a196d5b0a5c65105664901f810", - "url": "https://files.pythonhosted.org/packages/ed/30/cc6081090e0653b8bfeac45b5973027771050813c4ac167a277f4f355242/pydantic-1.9.1-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15", + "url": "https://files.pythonhosted.org/packages/f7/76/4a98738c31e520c78a80e9575b655b5c3ae96313102478bff4d643abc2e9/pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "f0f047e11febe5c3198ed346b507e1d010330d56ad615a7e0a89fae604065a0e", - "url": "https://files.pythonhosted.org/packages/f1/ea/1b879b2b30c4e041bce20f3baf785a640f93ae4146d5258dff5661d0bebf/pydantic-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d", + "url": "https://files.pythonhosted.org/packages/fd/8f/3f7e88b507dbdfec8f1f914294aa8831edffb03d668799c65b4b46331c8a/pydantic-1.9.2.tar.gz" } ], "project_name": "pydantic", @@ -1082,7 +1114,7 @@ "typing-extensions>=3.7.4.3" ], "requires_python": ">=3.6.1", - "version": "1.9.1" + "version": "1.9.2" }, { "artifacts": [ @@ -1437,13 +1469,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6", - "url": "https://files.pythonhosted.org/packages/90/2e/109766d7f3cb854a083dd66bfc0bf2bf9e40f373829efedb4b5e0d104aa3/setuptools-63.4.1-py3-none-any.whl" + "hash": "7f61f7e82647f77d4118eeaf43d64cbcd4d87e38af9611694d4866eb070cd10d", + "url": "https://files.pythonhosted.org/packages/2a/a3/49c29680d6118273b992b40ebe881e8e899b8e26a4e951f37f223da8f862/setuptools-63.4.3-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca", - "url": "https://files.pythonhosted.org/packages/63/2e/e1f3e1b02d7ff0baa356413e93ad8a88706accd6b3538e18204a8a00c42d/setuptools-63.4.1.tar.gz" + "hash": "521c833d1e5e1ef0869940e7f486a83de7773b9f029010ad0c2fe35453a9dad9", + "url": "https://files.pythonhosted.org/packages/5b/ff/69fd395c5237da934753752b71c38e95e137bd0603d5640df70ddaea8038/setuptools-63.4.3.tar.gz" } ], "project_name": "setuptools", @@ -1494,7 +1526,7 @@ "wheel; extra == \"testing-integration\"" ], "requires_python": ">=3.7", - "version": "63.4.1" + "version": "63.4.3" }, { "artifacts": [ @@ -2326,6 +2358,7 @@ "freezegun==1.2.1", "humbug==0.2.7", "ijson==3.1.4", + "importlib_resources==5.0.*", "mypy-typing-asserts==0.1.1", "packaging==21.3", "pex==2.1.103", diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index e09db9f10dd..62b108cc26d 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -13,7 +13,10 @@ python_test_utils(name="test_utils") python_distribution( name="pants-packaged", - dependencies=["./__main__.py", ":resources"], + dependencies=[ + "./__main__.py", + ":resources", + ], # Because we have native code, this will cause the wheel to use whatever the ABI is for the # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`. sdist=False, diff --git a/src/python/pants/backend/codegen/protobuf/scala/rules.py b/src/python/pants/backend/codegen/protobuf/scala/rules.py index 8591fa40c1c..65058f8b1b1 100644 --- a/src/python/pants/backend/codegen/protobuf/scala/rules.py +++ b/src/python/pants/backend/codegen/protobuf/scala/rules.py @@ -3,7 +3,6 @@ from __future__ import annotations import os -import pkgutil from dataclasses import dataclass from pants.backend.codegen import export_codegen_goal @@ -55,6 +54,7 @@ from pants.source.source_root import SourceRoot, SourceRootRequest from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource class GenerateScalaFromProtobufRequest(GenerateSourcesRequest): @@ -245,7 +245,7 @@ async def setup_scalapb_shim_classfiles( ) -> ScalaPBShimCompiledClassfiles: dest_dir = "classfiles" - scalapb_shim_content = pkgutil.get_data( + scalapb_shim_content = read_resource( "pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala" ) if not scalapb_shim_content: diff --git a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py index e5fecdef6ab..0b3f82acbab 100644 --- a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py +++ b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -35,6 +34,7 @@ from pants.engine.unions import UnionRule from pants.util.docutil import git_url from pants.util.logging import LogLevel +from pants.util.resources import read_resource _DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py" _DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems" @@ -77,7 +77,7 @@ class ParserSetup: @rule async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup: - parser_script_content = pkgutil.get_data(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) + parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) if not parser_script_content: raise ValueError( f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}." diff --git a/src/python/pants/backend/go/go_sources/load_go_binary.py b/src/python/pants/backend/go/go_sources/load_go_binary.py index 68c65369bd0..ca84f78e742 100644 --- a/src/python/pants/backend/go/go_sources/load_go_binary.py +++ b/src/python/pants/backend/go/go_sources/load_go_binary.py @@ -3,7 +3,6 @@ from __future__ import annotations -import pkgutil from dataclasses import dataclass from pants.backend.go.util_rules.build_pkg import BuildGoPackageRequest, BuiltGoPackage @@ -12,6 +11,7 @@ from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -31,7 +31,7 @@ def debug_hint(self) -> str: def setup_files(dir_name: str, file_names: tuple[str, ...]) -> tuple[FileContent, ...]: def get_file(file_name: str) -> bytes: - content = pkgutil.get_data(f"pants.backend.go.go_sources.{dir_name}", file_name) + content = read_resource(f"pants.backend.go.go_sources.{dir_name}", file_name) if not content: raise AssertionError(f"Unable to find resource for `{file_name}`.") return content diff --git a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py index 9dbb7f1fb23..c5107122692 100644 --- a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py +++ b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py @@ -4,7 +4,6 @@ import json import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator @@ -25,6 +24,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource _PARSER_KOTLIN_VERSION = "1.6.20" @@ -230,7 +230,7 @@ async def resolve_fallible_result_to_analysis( async def setup_kotlin_parser_classfiles(jdk: InternalJdk) -> KotlinParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.kotlin.dependency_inference", "KotlinParser.kt" ) if not parser_source_content: diff --git a/src/python/pants/backend/python/dependency_inference/BUILD b/src/python/pants/backend/python/dependency_inference/BUILD index 0290569f3ec..a7935536dbc 100644 --- a/src/python/pants/backend/python/dependency_inference/BUILD +++ b/src/python/pants/backend/python/dependency_inference/BUILD @@ -1,7 +1,11 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + dependencies=[ + "./scripts:dependency_parser", + ], +) python_tests( name="tests", diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index 183ded7992f..96b897397b9 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import json -import pkgutil from dataclasses import dataclass from pants.backend.python.target_types import PythonSourceField @@ -16,6 +15,7 @@ from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -61,7 +61,7 @@ class ParserScript: @rule async def parser_script() -> ParserScript: - script = pkgutil.get_data(__name__, "scripts/dependency_parser.py") + script = read_resource(__name__, "scripts/dependency_parser_py") assert script is not None return ParserScript( await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/BUILD b/src/python/pants/backend/python/dependency_inference/scripts/BUILD index e142aded947..8fd8d4d34e0 100644 --- a/src/python/pants/backend/python/dependency_inference/scripts/BUILD +++ b/src/python/pants/backend/python/dependency_inference/scripts/BUILD @@ -1,11 +1,15 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -resource(name="dependency_parser", source="dependency_parser.py") +resource( + name="dependency_parser", + source="dependency_parser_py", +) + # Also expose scripts as python sources so they get formatted/linted/checked. python_source( name="dependency_parser_source", - source="dependency_parser.py", + source="dependency_parser_py", # This is run with Python 2.7 and 3.5+, so we shouldn't be running pyupgrade. # skip_pyupgrade=True, ) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py b/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py similarity index 100% rename from src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py rename to src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser.py b/src/python/pants/backend/scala/dependency_inference/scala_parser.py index f119a6f78fa..09107b82813 100644 --- a/src/python/pants/backend/scala/dependency_inference/scala_parser.py +++ b/src/python/pants/backend/scala/dependency_inference/scala_parser.py @@ -5,7 +5,6 @@ import json import logging import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator, Mapping @@ -39,6 +38,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource logger = logging.getLogger(__name__) @@ -316,7 +316,7 @@ async def resolve_fallible_result_to_analysis( async def setup_scala_parser_classfiles(jdk: InternalJdk) -> ScalaParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.scala.dependency_inference", "ScalaParser.scala" ) if not parser_source_content: diff --git a/src/python/pants/backend/terraform/dependency_inference.py b/src/python/pants/backend/terraform/dependency_inference.py index 49dd76c0d7c..c6d7a015c9d 100644 --- a/src/python/pants/backend/terraform/dependency_inference.py +++ b/src/python/pants/backend/terraform/dependency_inference.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -36,6 +35,7 @@ from pants.util.docutil import git_url from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet +from pants.util.resources import read_resource class TerraformHcl2Parser(PythonToolRequirementsBase): @@ -75,7 +75,7 @@ class ParserSetup: @rule async def setup_parser(hcl2_parser: TerraformHcl2Parser) -> ParserSetup: - parser_script_content = pkgutil.get_data("pants.backend.terraform", "hcl2_parser.py") + parser_script_content = read_resource("pants.backend.terraform", "hcl2_parser.py") if not parser_script_content: raise ValueError("Unable to find source to hcl2_parser.py wrapper script.") diff --git a/src/python/pants/util/resources.py b/src/python/pants/util/resources.py new file mode 100644 index 00000000000..6effe6ddfdc --- /dev/null +++ b/src/python/pants/util/resources.py @@ -0,0 +1,28 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +import importlib +from itertools import chain + +import importlib_resources as resources + + +def read_resource(package_or_module: str, resource: str) -> bytes: + """Reads a resource file from within the Pants package itself. + + This helper function is designed for compatibility with `pkgutil.get_data()` wherever possible, + but also allows compability with PEP302 pluggable importers such as included with PyOxidizer. + """ + + a = importlib.import_module(package_or_module) + package_: str = a.__package__ # type: ignore[assignment] + resource_parts = resource.split("/") + + if len(resource_parts) == 1: + package = package_ + else: + package = ".".join(chain((package_,), resource_parts[:-1])) + resource = resource_parts[-1] + + return resources.read_binary(package, resource) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index adee488c8b5..57569a30faf 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -2,10 +2,12 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import pkgutil from packaging.version import Version +# Generate a inferrable dependency on the `pants._version` package and its associated resources. +from pants.util.resources import read_resource + # Set this env var to override the version pants reports. Useful for testing. _PANTS_VERSION_OVERRIDE = "_PANTS_VERSION_OVERRIDE" @@ -14,7 +16,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - pkgutil.get_data(__name__, "VERSION").decode().strip() # type: ignore[union-attr] + read_resource(__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION)