remote: Return exit code 34 for remote caching/execution errors.

For any errors that are due to failures in the remote caching /
execution layers Bazel now returns exit code 34 (ExitCode.REMOTE_ERROR).
This includes errors where the remote cache / executor is unreachable or
crashes. It does not include errors if the test / build failure is due
to user errors i.e. compilation or test failures.

PiperOrigin-RevId: 167259236

WORKSPACE

@@ -74,14 +74,14 @@ bind(
 
 new_local_repository(
     name = "com_google_protobuf",
-    build_file = "./third_party/protobuf/3.4.0/BUILD",
-    path = "./third_party/protobuf/3.4.0/",
+    build_file = "./third_party/protobuf/3.2.0/BUILD",
+    path = "./third_party/protobuf/3.2.0/",
 )
 
 new_local_repository(
     name = "com_google_protobuf_java",
-    build_file = "./third_party/protobuf/3.4.0/com_google_protobuf_java.BUILD",
-    path = "./third_party/protobuf/3.4.0/",
+    build_file = "./third_party/protobuf/3.2.0/com_google_protobuf_java.BUILD",
+    path = "./third_party/protobuf/3.2.0/",
 )
 
 new_local_repository(

src/main/java/com/google/devtools/build/lib/exec/SpawnResult.java

@@ -93,6 +93,12 @@ public enum Status {
      */
     REMOTE_EXECUTOR_OVERLOADED,
 
+    /**
+     * The result of the remotely executed Spawn could not be retrieved due to errors in the remote
+     * caching layer.
+     */
+    REMOTE_CACHE_FAILED,
+
     /**
      * The remote execution system did not allow the request due to missing authorization or
      * authentication.

src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java

@@ -27,12 +27,14 @@
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.exec.SpawnExecException;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnResult.Status;
 import com.google.devtools.build.lib.exec.SpawnRunner;
 import com.google.devtools.build.lib.remote.Digests.ActionKey;
 import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
+import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.util.io.FileOutErr;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
@@ -209,40 +211,40 @@ private SpawnResult execLocallyOrFail(Spawn spawn, SpawnExecutionPolicy policy,
     if (options.remoteLocalFallback) {
       return execLocally(spawn, policy, inputMap, uploadLocalResults, remoteCache, actionKey);
     }
-    if (cause instanceof TimeoutException) {
-      return handleTimeout(policy.getFileOutErr());
-    }
-    throw new EnvironmentalExecException(errorMessage(cause), cause, true);
-  }
-
-  private SpawnResult handleTimeout(FileOutErr outErr) throws IOException {
-    // TODO(buchgr): Once the remote execution protocol allows it, also provide stdout/stderr
-    // from the action that timed out.
-    try (OutputStream out = outErr.getOutputStream()) {
-      // This is a hack to ensure that the test.log file gets created, as else the action
-      // will complain that one of its outputs has not been created.
-      String msg = "Log output for timeouts is not yet supported in remote execution.";
-      out.write(msg.getBytes(StandardCharsets.UTF_8));
-      out.flush();
-    }
-    return new SpawnResult.Builder().setStatus(Status.TIMEOUT).build();
+    return handleError(cause, policy.getFileOutErr());
   }
 
-  private String errorMessage(IOException e) {
-    String message = "";
-    if (e instanceof RetryException
-        && ((RetryException) e).causedByStatusCode(Code.UNAVAILABLE)) {
-      message = "The remote executor/cache is unavailable";
-    } else if (e instanceof CacheNotFoundException) {
-      message = "Failed to download from remote cache";
+  private SpawnResult handleError(IOException cause, FileOutErr outErr) throws IOException,
+      ExecException {
+    final Status status;
+    if (cause instanceof TimeoutException) {
+      status = Status.TIMEOUT;
+      // TODO(buchgr): Once the remote execution protocol allows it, also provide stdout/stderr
+      // from the action that timed out.
+      try (OutputStream out = outErr.getOutputStream()) {
+        // This is a hack to ensure that the test.log file gets created, as else the action
+        // will complain that one of its outputs has not been created.
+        String msg = "Log output for timeouts is not yet supported in remote execution.";
+        out.write(msg.getBytes(StandardCharsets.UTF_8));
+        out.flush();
+      }
+      return new SpawnResult.Builder().setStatus(status).build();
+    } else if (cause instanceof RetryException
+        && ((RetryException) cause).causedByStatusCode(Code.UNAVAILABLE)) {
+      status = Status.CONNECTION_FAILED;
+    } else if (cause instanceof CacheNotFoundException) {
+      status = Status.REMOTE_CACHE_FAILED;
     } else {
-      message = "Error in remote cache/executor";
-    }
-    // TODO(olaola): reuse the ErrorMessage class for these errors.
-    if (verboseFailures) {
-      message += "\n" + Throwables.getStackTraceAsString(e);
+      status = Status.EXECUTION_FAILED;
     }
-    return message;
+    throw new SpawnExecException(
+        Throwables.getStackTraceAsString(cause),
+        new SpawnResult
+            .Builder()
+            .setExitCode(ExitCode.REMOTE_ERROR.getNumericExitCode())
+            .setStatus(status)
+            .build(),
+        /*catastrophic=*/true);
   }
 
   static Action buildAction(

src/main/protobuf/BUILD

@@ -2,7 +2,7 @@ package(default_visibility = ["//visibility:public"])
 
 load("//tools/build_rules:genproto.bzl", "cc_grpc_library")
 load("//tools/build_rules:utilities.bzl", "java_library_srcs")
-load("//third_party/protobuf/3.4.0:protobuf.bzl", "cc_proto_library", "py_proto_library")
+load("//third_party/protobuf/3.2.0:protobuf.bzl", "cc_proto_library", "py_proto_library")
 load("//third_party/grpc:build_defs.bzl", "java_grpc_library")
 
 FILES = [

src/test/java/com/google/devtools/build/lib/BUILD

@@ -1082,6 +1082,7 @@ java_test(
         "//src/main/java/com/google/devtools/build/lib:auth_and_tls_options",
         "//src/main/java/com/google/devtools/build/lib:build-base",
         "//src/main/java/com/google/devtools/build/lib:events",
+        "//src/main/java/com/google/devtools/build/lib:exitcode-external",
         "//src/main/java/com/google/devtools/build/lib:inmemoryfs",
         "//src/main/java/com/google/devtools/build/lib:io",
         "//src/main/java/com/google/devtools/build/lib:preconditions",

src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java

@@ -31,11 +31,12 @@
 import com.google.devtools.build.lib.actions.ActionInputHelper;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
-import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ResourceSet;
 import com.google.devtools.build.lib.actions.SimpleSpawn;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.authandtls.GrpcUtils;
+import com.google.devtools.build.lib.exec.SpawnExecException;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;
@@ -636,13 +637,12 @@ public void getActionResult(
     try {
       client.exec(simpleSpawn, simplePolicy);
       fail("Expected an exception");
-    } catch (EnvironmentalExecException expected) {
-      assertThat(expected).hasMessageThat().contains("The remote executor/cache is unavailable");
+    } catch (SpawnExecException expected) {
+      assertThat(expected.getSpawnResult().status())
+          .isEqualTo(SpawnResult.Status.CONNECTION_FAILED);
       // Ensure we also got back the stack trace.
       assertThat(expected).hasMessageThat()
           .contains("GrpcRemoteExecutionClientTest.passUnavailableErrorWithStackTrace");
-      Throwable t = expected.getCause();
-      assertThat(t).isInstanceOf(RetryException.class);
     }
   }
 
@@ -660,14 +660,11 @@ public void getActionResult(
     try {
       client.exec(simpleSpawn, simplePolicy);
       fail("Expected an exception");
-    } catch (EnvironmentalExecException expected) {
-      assertThat(expected).hasMessageThat().contains("Error in remote cache/executor");
+    } catch (ExecException expected) {
       assertThat(expected).hasMessageThat().contains("whoa"); // Error details.
       // Ensure we also got back the stack trace.
       assertThat(expected).hasMessageThat()
           .contains("GrpcRemoteExecutionClientTest.passInternalErrorWithStackTrace");
-      Throwable t = expected.getCause();
-      assertThat(t).isInstanceOf(RetryException.class);
     }
   }
 
@@ -719,14 +716,13 @@ public void read(ReadRequest request, StreamObserver<ReadResponse> responseObser
     try {
       client.exec(simpleSpawn, simplePolicy);
       fail("Expected an exception");
-    } catch (EnvironmentalExecException expected) {
-      assertThat(expected).hasMessageThat().contains("Failed to download from remote cache");
+    } catch (SpawnExecException expected) {
+      assertThat(expected.getSpawnResult().status())
+          .isEqualTo(SpawnResult.Status.REMOTE_CACHE_FAILED);
+      assertThat(expected).hasMessageThat().contains(stdOutDigest.getHash());
       // Ensure we also got back the stack trace.
       assertThat(expected).hasMessageThat()
           .contains("GrpcRemoteExecutionClientTest.passCacheMissErrorWithStackTrace");
-      Throwable t = expected.getCause();
-      assertThat(t).isInstanceOf(CacheNotFoundException.class);
-      assertThat(((CacheNotFoundException) t).getMissingDigest()).isEqualTo(stdOutDigest);
     }
   }
 
@@ -800,11 +796,9 @@ public void findMissingBlobs(
     try {
       client.exec(simpleSpawn, simplePolicy);
       fail("Expected an exception");
-    } catch (EnvironmentalExecException expected) {
-      assertThat(expected).hasMessageThat().contains("Failed to download from remote cache");
-      Throwable t = expected.getCause();
-      assertThat(t).isInstanceOf(CacheNotFoundException.class);
-      assertThat(((CacheNotFoundException) t).getMissingDigest()).isEqualTo(stdOutDigest);
+    } catch (ExecException expected) {
+      assertThat(expected).hasMessageThat().contains("Missing digest");
+      assertThat(expected).hasMessageThat().contains("476d9ec701e2de6a6c37ab5211117a7cb8333a27");
     }
   }
 }