diff --git a/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/BUILD b/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/BUILD new file mode 100644 index 00000000000..68aaac88424 --- /dev/null +++ b/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/BUILD @@ -0,0 +1,4 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() diff --git a/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/__init__.py b/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/register.py b/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/register.py new file mode 100644 index 00000000000..c58c426ab38 --- /dev/null +++ b/src/python/pants/backend/python/providers/experimental/pyenv/custom_install/register.py @@ -0,0 +1,14 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +from pants.backend.python.providers.pyenv.custom_install.rules import rules as custom_install_rules +from pants.backend.python.providers.pyenv.custom_install.target_types import PyenvInstall + + +def target_types(): + return [PyenvInstall] + + +def rules(): + return custom_install_rules() diff --git a/src/python/pants/backend/python/providers/experimental/pyenv/register.py b/src/python/pants/backend/python/providers/experimental/pyenv/register.py index c3283fc85aa..55259c45f7d 100644 --- a/src/python/pants/backend/python/providers/experimental/pyenv/register.py +++ b/src/python/pants/backend/python/providers/experimental/pyenv/register.py @@ -3,11 +3,6 @@ from pants.backend.python.providers.pyenv.rules import rules as pyenv_rules -from pants.backend.python.providers.pyenv.target_types import PyenvInstall - - -def target_types(): - return [PyenvInstall] def rules(): diff --git a/src/python/pants/backend/python/providers/pyenv/custom_install/BUILD b/src/python/pants/backend/python/providers/pyenv/custom_install/BUILD new file mode 100644 index 00000000000..2c11993a51f --- /dev/null +++ b/src/python/pants/backend/python/providers/pyenv/custom_install/BUILD @@ -0,0 +1,12 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() +python_tests( + name="tests", + overrides={ + "rules_integration_test.py": { + "timeout": 600, + } + }, +) diff --git a/src/python/pants/backend/python/providers/pyenv/custom_install/__init__.py b/src/python/pants/backend/python/providers/pyenv/custom_install/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/providers/pyenv/custom_install/rules.py b/src/python/pants/backend/python/providers/pyenv/custom_install/rules.py new file mode 100644 index 00000000000..6f980f3d33d --- /dev/null +++ b/src/python/pants/backend/python/providers/pyenv/custom_install/rules.py @@ -0,0 +1,102 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import dataclasses +from dataclasses import dataclass +from textwrap import dedent # noqa: PNT20 + +from pants.backend.python.providers.pyenv.custom_install.target_types import ( + PyenvInstallSentinelField, +) +from pants.backend.python.providers.pyenv.rules import ( + PyenvInstallInfoRequest, + PyenvPythonProviderSubsystem, +) +from pants.backend.python.providers.pyenv.rules import rules as pyenv_rules +from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior, RunRequest +from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest +from pants.core.util_rules.external_tool import rules as external_tools_rules +from pants.engine.fs import CreateDigest, FileContent +from pants.engine.internals.native_engine import Digest, MergeDigests +from pants.engine.internals.selectors import Get, MultiGet +from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest +from pants.engine.internals.target_adaptor import TargetAdaptor +from pants.engine.platform import Platform +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.frozendict import FrozenDict + + +@dataclass(frozen=True) +class SyntheticPyenvTargetsRequest(SyntheticTargetsRequest): + path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS + + +@rule +async def make_synthetic_targets(request: SyntheticPyenvTargetsRequest) -> SyntheticAddressMaps: + return SyntheticAddressMaps.for_targets_request( + request, [("BUILD.pyenv", (TargetAdaptor("_pyenv_install", "pants-pyenv-install"),))] + ) + + +@dataclass(frozen=True) +class RunPyenvInstallFieldSet(RunFieldSet): + run_in_sandbox_behavior = RunInSandboxBehavior.NOT_SUPPORTED + required_fields = (PyenvInstallSentinelField,) + + _sentinel: PyenvInstallSentinelField + + +@rule +async def run_pyenv_install( + _: RunPyenvInstallFieldSet, + platform: Platform, + pyenv_subsystem: PyenvPythonProviderSubsystem, +) -> RunRequest: + run_request, pyenv = await MultiGet( + Get(RunRequest, PyenvInstallInfoRequest()), + Get(DownloadedExternalTool, ExternalToolRequest, pyenv_subsystem.get_request(platform)), + ) + + wrapper_script_digest = await Get( + Digest, + CreateDigest( + [ + FileContent( + "run_install_python_shim.sh", + dedent( + f"""\ + #!/usr/bin/env bash + set -e + cd "$CHROOT" + SPECIFIC_VERSION=$("{pyenv.exe}" latest --known $1) + {" ".join(run_request.args)} $SPECIFIC_VERSION + """ + ).encode(), + is_executable=True, + ) + ] + ), + ) + digest = await Get(Digest, MergeDigests([run_request.digest, wrapper_script_digest])) + return dataclasses.replace( + run_request, + args=("{chroot}/run_install_python_shim.sh",), + digest=digest, + extra_env=FrozenDict( + { + "CHROOT": "{chroot}", + **run_request.extra_env, + } + ), + ) + + +def rules(): + return ( + *collect_rules(), + *external_tools_rules(), + *pyenv_rules(), + *RunPyenvInstallFieldSet.rules(), + UnionRule(SyntheticTargetsRequest, SyntheticPyenvTargetsRequest), + ) diff --git a/src/python/pants/backend/python/providers/pyenv/custom_install/rules_integration_test.py b/src/python/pants/backend/python/providers/pyenv/custom_install/rules_integration_test.py new file mode 100644 index 00000000000..ad14465123c --- /dev/null +++ b/src/python/pants/backend/python/providers/pyenv/custom_install/rules_integration_test.py @@ -0,0 +1,123 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.python import target_types_rules +from pants.backend.python.dependency_inference import rules as dependency_inference_rules +from pants.backend.python.goals.run_python_source import PythonSourceFieldSet +from pants.backend.python.goals.run_python_source import rules as run_rules +from pants.backend.python.providers.pyenv.custom_install.rules import RunPyenvInstallFieldSet +from pants.backend.python.providers.pyenv.custom_install.rules import ( + rules as pyenv_custom_install_rules, +) +from pants.backend.python.providers.pyenv.custom_install.target_types import PyenvInstall +from pants.backend.python.target_types import PythonSourcesGeneratorTarget +from pants.build_graph.address import Address +from pants.core.goals.run import RunRequest +from pants.engine.process import InteractiveProcess +from pants.engine.rules import QueryRule +from pants.engine.target import Target +from pants.testutil.rule_runner import RuleRunner, mock_console + + +@pytest.fixture +def named_caches_dir(tmp_path): + return f"{tmp_path}/named_cache" + + +@pytest.fixture +def rule_runner(named_caches_dir) -> RuleRunner: + return RuleRunner( + rules=[ + *run_rules(), + *pyenv_custom_install_rules(), + *dependency_inference_rules.rules(), + *target_types_rules.rules(), + QueryRule(RunRequest, (PythonSourceFieldSet,)), + QueryRule(RunRequest, (RunPyenvInstallFieldSet,)), + ], + target_types=[ + PythonSourcesGeneratorTarget, + PyenvInstall, + ], + bootstrap_args=[ + f"--named-caches-dir={named_caches_dir}", + ], + ) + + +def run_run_request( + rule_runner: RuleRunner, + target: Target, +) -> str: + args = [ + ( + "--backend-packages=[" + + "'pants.backend.python'," + + "'pants.backend.python.providers.experimental.pyenv'," + + "'pants.backend.python.providers.experimental.pyenv.custom_install'," + + "]" + ), + "--source-root-patterns=['src']", + ] + # Run the install + install_target = rule_runner.get_target( + Address(target_name="pants-pyenv-install", spec_path="") + ) + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + run_request = rule_runner.request(RunRequest, [RunPyenvInstallFieldSet.create(install_target)]) + run_process = InteractiveProcess( + argv=run_request.args + ("3.9.16",), + env=run_request.extra_env, + input_digest=run_request.digest, + run_in_workspace=True, + immutable_input_digests=run_request.immutable_input_digests, + append_only_caches=run_request.append_only_caches, + ) + with mock_console(rule_runner.options_bootstrapper) as mocked_console: + rule_runner.run_interactive_process(run_process) + print(mocked_console[1].get_stdout().strip()) + print(mocked_console[1].get_stderr().strip()) + assert "pyenv/versions/3.9.16/bin/python" in mocked_console[1].get_stdout().strip() + + run_request = rule_runner.request(RunRequest, [PythonSourceFieldSet.create(target)]) + run_process = InteractiveProcess( + argv=run_request.args, + env=run_request.extra_env, + input_digest=run_request.digest, + run_in_workspace=True, + immutable_input_digests=run_request.immutable_input_digests, + append_only_caches=run_request.append_only_caches, + ) + with mock_console(rule_runner.options_bootstrapper) as mocked_console: + rule_runner.run_interactive_process(run_process) + return mocked_console[1].get_stdout().strip() + + +def test_custom_install(rule_runner, named_caches_dir): + rule_runner.write_files( + { + "src/app.py": dedent( + """\ + import os.path + import sys + import sysconfig + + print(sysconfig.get_config_var("prefix")) + print(sys.version.replace("\\n", " ")) + """ + ), + "src/BUILD": "python_sources(interpreter_constraints=['==3.9.16'])", + } + ) + + target = rule_runner.get_target(Address("src", relative_file_path="app.py")) + stdout = run_run_request(rule_runner, target) + prefix_dir, version = stdout.splitlines() + assert prefix_dir.startswith(f"{named_caches_dir}/pyenv") + assert "3.9.16" in version diff --git a/src/python/pants/backend/python/providers/pyenv/target_types.py b/src/python/pants/backend/python/providers/pyenv/custom_install/target_types.py similarity index 100% rename from src/python/pants/backend/python/providers/pyenv/target_types.py rename to src/python/pants/backend/python/providers/pyenv/custom_install/target_types.py diff --git a/src/python/pants/backend/python/providers/pyenv/rules.py b/src/python/pants/backend/python/providers/pyenv/rules.py index d7214631f83..5fe54c7da5b 100644 --- a/src/python/pants/backend/python/providers/pyenv/rules.py +++ b/src/python/pants/backend/python/providers/pyenv/rules.py @@ -1,16 +1,13 @@ # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import dataclasses -from dataclasses import dataclass from textwrap import dedent # noqa: PNT20 -from pants.backend.python.providers.pyenv.target_types import PyenvInstallSentinelField from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.util_rules.pex import PythonProvider from pants.backend.python.util_rules.pex import rules as pex_rules from pants.backend.python.util_rules.pex_environment import PythonExecutable -from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior, RunRequest +from pants.core.goals.run import RunRequest from pants.core.util_rules.external_tool import ( DownloadedExternalTool, ExternalToolRequest, @@ -22,8 +19,6 @@ from pants.engine.fs import CreateDigest, FileContent from pants.engine.internals.native_engine import Digest, MergeDigests from pants.engine.internals.selectors import Get, MultiGet -from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest -from pants.engine.internals.target_adaptor import TargetAdaptor from pants.engine.platform import Platform from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, rule @@ -262,77 +257,10 @@ async def get_python( ) -@dataclass(frozen=True) -class SyntheticPyenvTargetsRequest(SyntheticTargetsRequest): - path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS - - -@rule -async def make_synthetic_targets(request: SyntheticPyenvTargetsRequest) -> SyntheticAddressMaps: - return SyntheticAddressMaps.for_targets_request( - request, [("BUILD.pyenv", (TargetAdaptor("_pyenv_install", "pants-pyenv-install"),))] - ) - - -@dataclass(frozen=True) -class RunPyenvInstallFieldSet(RunFieldSet): - run_in_sandbox_behavior = RunInSandboxBehavior.NOT_SUPPORTED - required_fields = (PyenvInstallSentinelField,) - - _sentinel: PyenvInstallSentinelField - - -@rule -async def run_pyenv_install( - _: RunPyenvInstallFieldSet, - platform: Platform, - pyenv_subsystem: PyenvPythonProviderSubsystem, -) -> RunRequest: - run_request, pyenv = await MultiGet( - Get(RunRequest, PyenvInstallInfoRequest()), - Get(DownloadedExternalTool, ExternalToolRequest, pyenv_subsystem.get_request(platform)), - ) - - wrapper_script_digest = await Get( - Digest, - CreateDigest( - [ - FileContent( - "run_install_python_shim.sh", - dedent( - f"""\ - #!/usr/bin/env bash - set -e - cd "$CHROOT" - SPECIFIC_VERSION=$("{pyenv.exe}" latest --known $1) - {" ".join(run_request.args)} $SPECIFIC_VERSION - """ - ).encode(), - is_executable=True, - ) - ] - ), - ) - digest = await Get(Digest, MergeDigests([run_request.digest, wrapper_script_digest])) - return dataclasses.replace( - run_request, - args=("{chroot}/run_install_python_shim.sh",), - digest=digest, - extra_env=FrozenDict( - { - "CHROOT": "{chroot}", - **run_request.extra_env, - } - ), - ) - - def rules(): return ( *collect_rules(), *pex_rules(), *external_tools_rules(), - *RunPyenvInstallFieldSet.rules(), UnionRule(PythonProvider, PyenvPythonProvider), - UnionRule(SyntheticTargetsRequest, SyntheticPyenvTargetsRequest), ) diff --git a/src/python/pants/backend/python/providers/pyenv/rules_integration_test.py b/src/python/pants/backend/python/providers/pyenv/rules_integration_test.py index 3182e05ab2b..365cb92b739 100644 --- a/src/python/pants/backend/python/providers/pyenv/rules_integration_test.py +++ b/src/python/pants/backend/python/providers/pyenv/rules_integration_test.py @@ -13,7 +13,6 @@ from pants.backend.python.goals.run_python_source import PythonSourceFieldSet from pants.backend.python.goals.run_python_source import rules as run_rules from pants.backend.python.providers.pyenv.rules import rules as pyenv_rules -from pants.backend.python.providers.pyenv.target_types import PyenvInstall from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.build_graph.address import Address from pants.core.goals.run import RunRequest @@ -35,9 +34,7 @@ def rule_runner() -> RuleRunner: ], target_types=[ PythonSourcesGeneratorTarget, - PyenvInstall, ], - preserve_tmpdirs=True, ) @@ -60,9 +57,7 @@ def run_run_request( append_only_caches=run_request.append_only_caches, ) with mock_console(rule_runner.options_bootstrapper) as mocked_console: - exit_code = rule_runner.run_interactive_process(run_process) - if exit_code: - print(mocked_console[1].get_stderr().strip()) + rule_runner.run_interactive_process(run_process) return mocked_console[1].get_stdout().strip()