Add (experimental) named caches field to shell_command and `adhoc_t…

…ool` (pantsbuild#19915)

Part of the effort towards making Pants compile its engine without
starting from scratch, and a feature we've probably all mused at some
point.

Marked `experimental` because it's completely wild-west right now.

src/python/pants/backend/adhoc/adhoc_tool.py

@@ -9,6 +9,7 @@
     AdhocToolExecutionDependenciesField,
     AdhocToolExtraEnvVarsField,
     AdhocToolLogOutputField,
+    AdhocToolNamedCachesField,
     AdhocToolOutputDirectoriesField,
     AdhocToolOutputFilesField,
     AdhocToolOutputRootDirField,
@@ -149,6 +150,11 @@ async def run_in_sandbox_request(
 
     extra_args = target.get(AdhocToolArgumentsField).value or ()
 
+    append_only_caches = {
+        **merged_extras.append_only_caches,
+        **(target.get(AdhocToolNamedCachesField).value or {}),
+    }
+
     process_request = AdhocProcessRequest(
         description=description,
         address=target.address,
@@ -158,7 +164,7 @@ async def run_in_sandbox_request(
         timeout=None,
         input_digest=input_digest,
         immutable_input_digests=FrozenDict.frozen(merged_extras.immutable_input_digests),
-        append_only_caches=FrozenDict.frozen(merged_extras.append_only_caches),
+        append_only_caches=FrozenDict(append_only_caches),
         output_files=output_files,
         output_directories=output_directories,
         fetch_env_vars=target.get(AdhocToolExtraEnvVarsField).value or (),

src/python/pants/backend/adhoc/target_types.py

@@ -10,6 +10,7 @@
     COMMON_TARGET_FIELDS,
     BoolField,
     Dependencies,
+    DictStringToStringField,
     IntField,
     MultipleSourcesField,
     SpecialCasedDependencies,
@@ -216,6 +217,25 @@ class AdhocToolWorkdirField(StringField):
     )
 
 
+class AdhocToolNamedCachesField(DictStringToStringField):
+    alias = "experimental_named_caches"
+    help = help_text(
+        """
+        Named caches to construct for the execution.
+        See https://www.pantsbuild.org/docs/reference-global#named_caches_dir.
+
+        The keys of the mapping are the directory name to be created in the named caches dir.
+        The values are the name of the symlink (relative to the sandbox root) in the sandbox which
+        points to the subdirectory in the named caches dir
+
+        NOTE: The named caches MUST be handled with great care. Processes accessing the named caches
+        can be run in parallel, and can be cancelled at any point in their execution (and
+        potentially restarted). That means that _every_ operation modifying the contents of the cache
+        MUST be concurrency and cancellation safe.
+        """
+    )
+
+
 class AdhocToolOutputRootDirField(StringField):
     alias: ClassVar[str] = "root_output_directory"
     default = "/"

src/python/pants/backend/shell/target_types.py

@@ -11,6 +11,7 @@
     AdhocToolExecutionDependenciesField,
     AdhocToolExtraEnvVarsField,
     AdhocToolLogOutputField,
+    AdhocToolNamedCachesField,
     AdhocToolOutputDependenciesField,
     AdhocToolOutputDirectoriesField,
     AdhocToolOutputFilesField,
@@ -374,6 +375,10 @@ class ShellCommandTestDependenciesField(ShellCommandExecutionDependenciesField):
     pass
 
 
+class ShellCommandNamedCachesField(AdhocToolNamedCachesField):
+    pass
+
+
 class SkipShellCommandTestsField(BoolField):
     alias = "skip_tests"
     default = False
@@ -396,6 +401,7 @@ class ShellCommandTarget(Target):
         ShellCommandToolsField,
         ShellCommandExtraEnvVarsField,
         ShellCommandWorkdirField,
+        ShellCommandNamedCachesField,
         ShellCommandOutputRootDirField,
         EnvironmentField,
     )

src/python/pants/backend/shell/util_rules/shell_command.py

@@ -14,6 +14,7 @@
     ShellCommandExecutionDependenciesField,
     ShellCommandExtraEnvVarsField,
     ShellCommandLogOutputField,
+    ShellCommandNamedCachesField,
     ShellCommandOutputDirectoriesField,
     ShellCommandOutputFilesField,
     ShellCommandOutputRootDirField,
@@ -140,6 +141,11 @@ async def _prepare_process_request_from_target(
     if merged_extras.path:
         extra_env["PATH"] = merged_extras.path
 
+    append_only_caches = {
+        **merged_extras.append_only_caches,
+        **(shell_command.get(ShellCommandNamedCachesField).value or {}),
+    }
+
     return AdhocProcessRequest(
         description=description,
         address=shell_command.address,
@@ -151,7 +157,7 @@ async def _prepare_process_request_from_target(
         output_files=output_files,
         output_directories=output_directories,
         fetch_env_vars=shell_command.get(ShellCommandExtraEnvVarsField).value or (),
-        append_only_caches=FrozenDict.frozen(merged_extras.append_only_caches),
+        append_only_caches=FrozenDict(append_only_caches),
         supplied_env_var_values=FrozenDict(extra_env),
         immutable_input_digests=FrozenDict.frozen(merged_extras.immutable_input_digests),
         log_on_process_errors=_LOG_ON_PROCESS_ERRORS,