Allow targets that implement RunFieldSet to be executed in the sand…

…box for side-effects (pantsbuild#17716)

This adds `experimental_run_in_sandbox`, which allows any target implementing `RunFieldSet`/returning `RunRequest` to be run in the sandbox with its execution dependencies.

Amongst other things, this saves needing to explicitly declare a `pex_binary` in order to run a `python_source`, however, other languages aren't as complete right now (though we can work on that).

Tests to follow once the API is bedded down: right now, this is _highly_ experimental, and mostly piggybacks on `experimental_shell_command` infrastructure.

Closes pantsbuild#17405.


Co-authored-by: Huon Wilson <[email protected]>
Co-authored-by: Joshua Cannon <[email protected]>

.github/workflows/test.yaml

@@ -512,7 +512,7 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v3
       with:
-        fetch-depth: 10
+        fetch-depth: 2
     - id: files
       name: Get changed files
       uses: tj-actions/changed-files@v32

build-support/bin/generate_github_workflows.py

@@ -105,7 +105,8 @@ def classify_changes() -> Jobs:
                 "other": gha_expr("steps.classify.outputs.other"),
             },
             "steps": [
-                *checkout(),
+                # fetch_depth must be 2.
+                *checkout(fetch_depth=2),
                 {
                     "id": "files",
                     "name": "Get changed files",
@@ -158,7 +159,7 @@ def ensure_category_label() -> Sequence[Step]:
     ]
 
 
-def checkout(*, containerized: bool = False) -> Sequence[Step]:
+def checkout(*, fetch_depth: int = 10, containerized: bool = False) -> Sequence[Step]:
     """Get prior commits and the commit message."""
     steps = [
         # See https://github.community/t/accessing-commit-message-in-pull-request-event/17158/8
@@ -167,7 +168,7 @@ def checkout(*, containerized: bool = False) -> Sequence[Step]:
         {
             "name": "Check out code",
             "uses": "actions/checkout@v3",
-            "with": {"fetch-depth": 10},
+            "with": {"fetch-depth": fetch_depth},
         },
     ]
     if containerized:

docs/markdown/Writing Plugins/common-plugin-tasks/plugin-upgrade-guide.md

@@ -25,6 +25,20 @@ def rules():
 
 Additionally, these types now by-default register the implementations for the rules used for `--debug`/`--debug-adapter`. If your plugin doesn't support these flags, simply remove the rules you've declared and let the default ones handle erroring. If your plugin does support these, set the class property(s) `supports_debug = True`/`supports_debug_adapter = True`, respectively.
 
+### `RunFieldSet` can be used to run targets in the sandbox as part of a build rule
+
+With the new `experimental_run_in_sandbox` target type, targets that implement `RunFieldSet` can be run as a build rule for their side-effects.
+
+Many rules that create `RunRequest`s can be used verbatim, but others may make assumptions that they will not be run hermetically. You will need set `run_in_sandbox_behavior` to one of the following values to generate a rule that allows your targets to be run in the sandbox:
+
+* `RunInSandboxBehavior.RUN_REQUEST_HERMETIC`: Use the existing `RunRequest`-generating rule, and enable cacheing. Use this if you are confident the behaviour of the rule relies only on state that is captured by pants (e.g. binary paths are found using `EnvironmentVarsRequest`), and that the rule only refers to files in the sandbox.
+* `RunInSandboxBehavior.RUN_REQUEST_NOT_HERMETIC`: Use the existing `RunRequest`-generating rule, and do not enable cacheing. Use this if your existing rule is mostly suitable for use in the sandbox, but you cannot guarantee reproducible behavior.
+* `RunInSandboxBehavior.CUSTOM`: Opt to write your own rule that returns `RunInSandboxRequest`.
+* `RunInSandboxBehavior.NOT_SUPPORTED`: Opt out of being usable in `experimental_run_in_sandbox`. Attempting to use such a target will result in a runtime exception.
+
+We expect to deprecate `RUN_REQUEST_NOT_HERMETIC` and `NOT_SUPPORTED` in a few versions time: these options are provided to give you some time to make your existing rules match the semantics of `RUN_REQUEST_HERMETIC`, or to add a `CUSTOM` rule.
+
+
 2.15
 ----
 

src/python/pants/backend/docker/goals/run_image.py

@@ -11,7 +11,7 @@
 from pants.backend.docker.target_types import DockerImageRegistriesField, DockerImageSourceField
 from pants.backend.docker.util_rules.docker_binary import DockerBinary
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
-from pants.core.goals.run import RunFieldSet, RunRequest
+from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior, RunRequest
 from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import WrappedTarget, WrappedTargetRequest
@@ -20,6 +20,7 @@
 @dataclass(frozen=True)
 class DockerRunFieldSet(RunFieldSet):
     required_fields = (DockerImageSourceField,)
+    run_in_sandbox_behavior = RunInSandboxBehavior.RUN_REQUEST_HERMETIC
 
 
 @rule

src/python/pants/backend/go/goals/package_binary.py

@@ -21,7 +21,7 @@
     OutputPathField,
     PackageFieldSet,
 )
-from pants.core.goals.run import RunFieldSet
+from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior
 from pants.engine.fs import AddPrefix, Digest, MergeDigests
 from pants.engine.internals.selectors import Get, MultiGet
 from pants.engine.rules import collect_rules, rule
@@ -32,6 +32,7 @@
 @dataclass(frozen=True)
 class GoBinaryFieldSet(PackageFieldSet, RunFieldSet):
     required_fields = (GoBinaryMainPackageField,)
+    run_in_sandbox_behavior = RunInSandboxBehavior.RUN_REQUEST_HERMETIC
 
     main: GoBinaryMainPackageField
     output_path: OutputPathField

src/python/pants/backend/python/goals/package_pex_binary.py

@@ -36,7 +36,7 @@
     OutputPathField,
     PackageFieldSet,
 )
-from pants.core.goals.run import RunFieldSet
+from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior
 from pants.core.target_types import FileSourceField
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import (
@@ -54,6 +54,8 @@
 
 @dataclass(frozen=True)
 class PexBinaryFieldSet(PackageFieldSet, RunFieldSet):
+    run_in_sandbox_behavior = RunInSandboxBehavior.RUN_REQUEST_HERMETIC
+
     required_fields = (PexEntryPointField,)
 
     entry_point: PexEntryPointField

src/python/pants/backend/python/goals/run_python_source.py

@@ -15,10 +15,18 @@
     PythonRunGoalUseSandboxField,
     PythonSourceField,
 )
+from pants.backend.python.target_types_rules import rules as python_target_types_rules
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_environment import PexEnvironment
-from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest
+from pants.backend.python.util_rules.pex_from_targets import rules as pex_from_targets_rules
+from pants.core.goals.run import (
+    RunDebugAdapterRequest,
+    RunFieldSet,
+    RunInSandboxBehavior,
+    RunInSandboxRequest,
+    RunRequest,
+)
 from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem
 from pants.engine.internals.selectors import Get
 from pants.engine.rules import collect_rules, rule
@@ -29,6 +37,7 @@
 class PythonSourceFieldSet(RunFieldSet):
     supports_debug_adapter = True
     required_fields = (PythonSourceField, PythonRunGoalUseSandboxField)
+    run_in_sandbox_behavior = RunInSandboxBehavior.CUSTOM
 
     source: PythonSourceField
     interpreter_constraints: InterpreterConstraintsField
@@ -52,6 +61,21 @@ async def create_python_source_run_request(
     )
 
 
+@rule(level=LogLevel.DEBUG)
+async def create_python_source_run_in_sandbox_request(
+    field_set: PythonSourceFieldSet, pex_env: PexEnvironment, python_setup: PythonSetup
+) -> RunInSandboxRequest:
+    # Unlike for `RunRequest`s, `run_in_sandbox` should _always_ be true when running in the
+    # sandbox.
+    run_request = await _create_python_source_run_request(
+        field_set.address,
+        entry_point_field=PexEntryPointField(field_set.source.value, field_set.address),
+        pex_env=pex_env,
+        run_in_sandbox=True,
+    )
+    return run_request.to_run_in_sandbox_request()
+
+
 @rule
 async def create_python_source_debug_adapter_request(
     field_set: PythonSourceFieldSet,
@@ -92,4 +116,6 @@ def rules():
     return [
         *collect_rules(),
         *PythonSourceFieldSet.rules(),
+        *pex_from_targets_rules(),
+        *python_target_types_rules(),
     ]

src/python/pants/backend/python/packaging/pyoxidizer/rules.py

@@ -24,7 +24,7 @@
 from pants.backend.python.target_types import GenerateSetupField, WheelField
 from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest
 from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet
-from pants.core.goals.run import RunFieldSet, RunRequest
+from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior, RunRequest
 from pants.core.util_rules.system_binaries import BashBinary
 from pants.engine.fs import (
     AddPrefix,
@@ -59,6 +59,7 @@
 @dataclass(frozen=True)
 class PyOxidizerFieldSet(PackageFieldSet, RunFieldSet):
     required_fields = (PyOxidizerDependenciesField,)
+    run_in_sandbox_behavior = RunInSandboxBehavior.RUN_REQUEST_HERMETIC
 
     binary_name: PyOxidizerBinaryNameField
     entry_point: PyOxidizerEntryPointField

src/python/pants/backend/shell/register.py

@@ -8,6 +8,7 @@
     ShellCommandRunTarget,
     ShellCommandTarget,
     ShellCommandTestTarget,
+    ShellRunInSandboxTarget,
     ShellSourcesGeneratorTarget,
     ShellSourceTarget,
     Shunit2TestsGeneratorTarget,
@@ -20,6 +21,7 @@
 def target_types():
     return [
         ShellCommandTarget,
+        ShellRunInSandboxTarget,
         ShellCommandRunTarget,
         ShellCommandTestTarget,
         ShellSourcesGeneratorTarget,

src/python/pants/backend/shell/target_types.py

@@ -260,6 +260,19 @@ class ShellCommandCommandField(StringField):
     help = "Shell command to execute.\n\nThe command is executed as 'bash -c <command>' by default."
 
 
+class RunInSandboxRunnableField(StringField):
+    alias = "runnable"
+    required = True
+    help = softwrap(
+        """
+        Address to a target that can be invoked by the `run` goal (and does not set
+        `run_in_sandbox_behavior=NOT_SUPPORTED`). This will be executed along with any arguments
+        specified by `argv`, in a sandbox with that target's transitive dependencies, along with
+        the transitive dependencies specified by `execution_dependencies`.
+        """
+    )
+
+
 class ShellCommandOutputsField(StringSequenceField):
     alias = "outputs"
     help = softwrap(
@@ -346,6 +359,19 @@ class ShellCommandSourcesField(MultipleSourcesField):
     expected_num_files = 0
 
 
+class RunInSandboxSourcesField(MultipleSourcesField):
+    # We solely register this field for codegen to work.
+    alias = "_sources"
+    uses_source_roots = False
+    expected_num_files = 0
+
+
+class RunInSandboxArgumentsField(StringSequenceField):
+    alias = "args"
+    default = ()
+    help = f"Extra arguments to pass into the `{RunInSandboxRunnableField.alias}` field."
+
+
 class ShellCommandTimeoutField(IntField):
     alias = "timeout"
     default = 30
@@ -442,6 +468,42 @@ class ShellCommandTarget(Target):
     )
 
 
+class ShellRunInSandboxTarget(Target):
+    alias = "experimental_run_in_sandbox"
+    core_fields = (
+        *COMMON_TARGET_FIELDS,
+        RunInSandboxRunnableField,
+        RunInSandboxArgumentsField,
+        ShellCommandExecutionDependenciesField,
+        ShellCommandOutputDependenciesField,
+        ShellCommandLogOutputField,
+        ShellCommandOutputFilesField,
+        ShellCommandOutputDirectoriesField,
+        RunInSandboxSourcesField,
+        ShellCommandTimeoutField,
+        ShellCommandToolsField,
+        ShellCommandExtraEnvVarsField,
+        EnvironmentField,
+    )
+    help = softwrap(
+        """
+        Execute any runnable target for its side effects.
+
+        Example BUILD file:
+
+            experimental_run_in_sandbox(
+                runnable=":python_source",
+                argv=[""],
+                tools=["tar", "curl", "cat", "bash", "env"],
+                execution_dependencies=[":scripts"],
+                outputs=["results/", "logs/my-script.log"],
+            )
+
+            shell_sources(name="scripts")
+        """
+    )
+
+
 class ShellCommandRunTarget(Target):
     alias = "experimental_run_shell_command"
     core_fields = (