add always mode for RemoteCacheWarningsBehavior (pantsbuild#20268)

Looked at the configuration docs here for a Slack thread, and felt like
this would've been the most sensible option to use -- but it did not
exist. 🤷

src/python/pants/option/global_options.py

@@ -334,6 +334,7 @@ class RemoteCacheWarningsBehavior(Enum):
     ignore = "ignore"
     first_only = "first_only"
     backoff = "backoff"
+    always = "always"
 
 
 @enum.unique

src/rust/engine/process_execution/remote/src/remote_cache.rs

@@ -37,6 +37,7 @@ pub enum RemoteCacheWarningsBehavior {
     Ignore,
     FirstOnly,
     Backoff,
+    Always,
 }
 
 pub struct RemoteCacheRunnerOptions {
@@ -424,19 +425,24 @@ impl CommandRunner {
             CacheErrorType::ReadError => "read from",
             CacheErrorType::WriteError => "write to",
         };
-        let log_msg = format!(
-            "Failed to {failure_desc} remote cache ({err_count} occurrences so far): {err}"
-        );
+
         let log_at_warn = match self.warnings_behavior {
             RemoteCacheWarningsBehavior::Ignore => false,
             RemoteCacheWarningsBehavior::FirstOnly => err_count == 1,
             RemoteCacheWarningsBehavior::Backoff => err_count.is_power_of_two(),
+            RemoteCacheWarningsBehavior::Always => true,
         };
-        if log_at_warn {
-            log::warn!("{}", log_msg);
+
+        let level = if log_at_warn {
+            Level::Warn
         } else {
-            log::debug!("{}", log_msg);
-        }
+            Level::Debug
+        };
+
+        log::log!(
+            level,
+            "Failed to {failure_desc} remote cache ({err_count} occurrences so far): {err}"
+        );
     }
 }
 

tests/python/pants_test/integration/remote_cache_integration_test.py

@@ -103,6 +103,17 @@ def write_err(i: int) -> str:
     for err in [third_read_err, third_write_err]:
         assert err not in backoff_result
 
+    always_result = run(RemoteCacheWarningsBehavior.always)
+    for err in [
+        first_read_err,
+        first_write_err,
+        third_read_err,
+        third_write_err,
+        fourth_read_err,
+        fourth_write_err,
+    ]:
+        assert err in always_result, f"Not found in:\n{always_result}"
+
 
 class ProcessOutputEntries(DigestEntries):
     pass