move creation of named cache directory into NamedCaches (pantsbuild#1…

…6685)

Complete a TODO and move creation of the destination directory for named caches into the `NamedCaches::local_paths` function. This is necessary to support the Docker command runner in pantsbuild#16670 which needs to modify the destination symlink to account for difference in paths between the host and container filesystems.

[ci skip-build-wheels]

src/rust/engine/process_execution/src/local.rs

@@ -17,8 +17,8 @@ use std::time::Instant;
 use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
 use fs::{
-  self, safe_create_dir_all_ioerror, DirectoryDigest, GlobExpansionConjunction, GlobMatching,
-  PathGlobs, Permissions, RelativePath, StrictGlobMatching, EMPTY_DIRECTORY_DIGEST,
+  self, DirectoryDigest, GlobExpansionConjunction, GlobMatching, PathGlobs, Permissions,
+  RelativePath, StrictGlobMatching, EMPTY_DIRECTORY_DIGEST,
 };
 use futures::future::{BoxFuture, FutureExt, TryFutureExt};
 use futures::stream::{BoxStream, StreamExt, TryStreamExt};
@@ -638,7 +638,16 @@ pub async fn prepare_workdir(
     .local_paths(&req.input_digests.immutable_inputs)
     .await?
     .into_iter()
-    .chain(named_caches.local_paths(&req.append_only_caches))
+    .chain(
+      named_caches
+        .local_paths(&req.append_only_caches)
+        .map_err(|err| {
+          StoreError::Unclassified(format!(
+            "Failed to make named cache(s) for local execution: {:?}",
+            err
+          ))
+        })?,
+    )
     .collect::<Vec<_>>();
 
   // Capture argv0 as the executable path so that we can test whether we have created it in the
@@ -702,14 +711,6 @@ pub async fn prepare_workdir(
       }
 
       for workdir_symlink in workdir_symlinks {
-        // TODO: Move initialization of the dst directory into NamedCaches.
-        safe_create_dir_all_ioerror(&workdir_symlink.dst).map_err(|err| {
-          format!(
-            "Error making {} for local execution: {:?}",
-            workdir_symlink.dst.display(),
-            err
-          )
-        })?;
         let src = workdir_path2.join(&workdir_symlink.src);
         symlink(&workdir_symlink.dst, &src).map_err(|err| {
           format!(

src/rust/engine/process_execution/src/named_caches.rs

@@ -5,7 +5,7 @@ use deepsize::DeepSizeOf;
 use serde::Serialize;
 
 use crate::WorkdirSymlink;
-use fs::{default_cache_path, RelativePath};
+use fs::{default_cache_path, safe_create_dir_all_ioerror, RelativePath};
 
 #[derive(Clone, Debug, DeepSizeOf, Eq, PartialEq, Hash, PartialOrd, Ord, Serialize)]
 pub struct CacheName(String);
@@ -51,13 +51,26 @@ impl NamedCaches {
   pub fn local_paths<'a>(
     &'a self,
     caches: &'a BTreeMap<CacheName, RelativePath>,
-  ) -> impl Iterator<Item = WorkdirSymlink> + 'a {
-    caches
+  ) -> Result<Vec<WorkdirSymlink>, String> {
+    let symlinks = caches
       .iter()
       .map(move |(cache_name, workdir_rel_path)| WorkdirSymlink {
         src: workdir_rel_path.clone(),
         dst: self.local_base.join(&cache_name.0),
       })
+      .collect::<Vec<_>>();
+
+    for symlink in &symlinks {
+      safe_create_dir_all_ioerror(&symlink.dst).map_err(|err| {
+        format!(
+          "Error creating directory {}: {:?}",
+          symlink.dst.display(),
+          err
+        )
+      })?
+    }
+
+    Ok(symlinks)
   }
 
   ///