[CI][Green-Ray][1] Automated retry of infra-error release tests (ray-…

…project#34057)

This PR is a part of my effort to make OSS release test run greener, starting with reducing infra error rates. Other work such as [this from Lonnie](https://docs.google.com/document/d/1hF7h8F19qFWFxH9WVeT8fWwVuNyUyHLTx-7LP3uxD50/edit#heading=h.i0cvl0u8jbfu) fixes systematic issues such as unstable Anyscale staging environment. This PR addresses transient issues with Anyscale that are hard to avoid in a distributed system. On a day Anyscale behaves well, transient issue seem to be around [2-3%](https://b534fd88.us1a.app.preset.io/superset/dashboard/43/?force=false&native_filters_key=MoYaGptJfGwbkF60A7RSzfoRLL_ypDf_JvNFxp2YGQ8Ls4CNgbAWEBh0WcOkOLsS), aka. 4 random failures for a test suite of 200 tests, annoying!

Concretely it will:

- First, classify an infra test run as a transient infra issue
- Instruct buildkite to automatically retry on transient issue
- If retry runs out, classify the infra test run as infra issue

Some other limitations that will be addressed in followup PRs:
- Move infra-failure retry configuration into LaunchDarkly?
- Limit auto-retry based on test cost or test runtime

Signed-off-by: Cuong Nguyen <[email protected]>

release/ray_release/buildkite/step.py

@@ -50,6 +50,14 @@
     ],
     "artifact_paths": [f"{DEFAULT_ARTIFACTS_DIR_HOST}/**/*"],
     "priority": 0,
+    "retry": {
+        "automatic": [
+            {
+                "exit_status": os.environ.get("BUILDKITE_RETRY_CODE", 79),
+                "limit": os.environ.get("BUILDKITE_MAX_RETRIES", 1),
+            }
+        ]
+    },
 }
 
 

release/ray_release/command_runner/command_runner.py

@@ -119,12 +119,12 @@ def run_prepare_command(
             max_retries=3,
         )
 
-    def get_last_logs(self) -> str:
+    def get_last_logs(self) -> Optional[str]:
         try:
             return self.get_last_logs_ex()
         except Exception as e:
             logger.exception(f"Error fetching logs: {e}")
-            return "No logs could be retrieved."
+            return None
 
     def get_last_logs_ex(self):
         raise NotImplementedError

release/ray_release/glue.py

@@ -1,5 +1,6 @@
 import os
 import time
+import traceback
 from typing import Optional, List, Tuple
 
 from ray_release.alerts.handle import handle_result, require_result
@@ -453,21 +454,21 @@ def run_release_test(
 ) -> Result:
     old_wd = os.getcwd()
     start_time = time.monotonic()
-
-    buildkite_group(":spiral_note_pad: Loading test configuration")
-    cluster_manager, command_runner, artifact_path = _load_test_configuration(
-        test,
-        anyscale_project,
-        result,
-        ray_wheels_url,
-        smoke_test,
-        no_terminate,
-    )
-
+    command_runner = None
+    cluster_manager = None
     pipeline_exception = None
     # non critical for some tests. So separate it from the general one.
     fetch_result_exception = None
     try:
+        buildkite_group(":spiral_note_pad: Loading test configuration")
+        cluster_manager, command_runner, artifact_path = _load_test_configuration(
+            test,
+            anyscale_project,
+            result,
+            ray_wheels_url,
+            smoke_test,
+            no_terminate,
+        )
         buildkite_group(":nut_and_bolt: Setting up cluster environment")
         (
             prepare_cmd,
@@ -525,7 +526,6 @@ def run_release_test(
             smoke_test,
             start_time_unix,
         )
-
     except Exception as e:
         logger.exception(e)
         buildkite_open_last()
@@ -540,9 +540,9 @@ def run_release_test(
         result.job_url = command_runner.job_manager.job_url
         result.job_id = command_runner.job_manager.job_id
 
-    result.last_logs = command_runner.get_last_logs()
+    result.last_logs = command_runner.get_last_logs() if command_runner else None
 
-    if not no_terminate:
+    if not no_terminate and cluster_manager:
         buildkite_group(":earth_africa: Terminating cluster")
         cluster_manager.terminate_cluster(wait=False)
 
@@ -570,12 +570,17 @@ def run_release_test(
 
     if pipeline_exception:
         buildkite_group(":rotating_light: Handling errors")
-        exit_code, error_type, runtime = handle_exception(pipeline_exception)
+        exit_code, result_status, runtime = handle_exception(pipeline_exception)
 
         result.return_code = exit_code.value
-        result.status = error_type
+        result.status = result_status.value
         if runtime is not None:
             result.runtime = runtime
+        try:
+            raise pipeline_exception
+        except Exception:
+            if not result.last_logs:
+                result.last_logs = traceback.format_exc()
 
     buildkite_group(":memo: Reporting results", open=True)
     for reporter in reporters or []:

release/ray_release/result.py

@@ -1,13 +1,29 @@
 import enum
+import os
 from dataclasses import dataclass
 from typing import Optional, Dict, Tuple
 
 
+class ResultStatus(enum.Enum):
+    """
+    Overall status of the result test run
+    """
+
+    SUCCESS = "success"
+    UNKNOWN = "unknown"
+    RUNTIME_ERROR = "runtime_error"
+    TRANSIENT_INFRA_ERROR = "transient_infra_error"
+    INFRA_ERROR = "infra_error"
+    INFRA_TIMEOUT = "infra_timeout"
+    ERROR = "error"
+    TIMEOUT = "timeout"
+
+
 @dataclass
 class Result:
     results: Optional[Dict] = None
 
-    status: str = "invalid"
+    status: str = ResultStatus.UNKNOWN.value
     return_code: int = 0
     last_logs: Optional[str] = None
 
@@ -64,33 +80,34 @@ class ExitCode(enum.Enum):
     PREPARE_ERROR = 43
 
 
-def handle_exception(e: Exception) -> Tuple[ExitCode, str, Optional[int]]:
+def handle_exception(e: Exception) -> Tuple[ExitCode, ResultStatus, Optional[int]]:
     from ray_release.exception import ReleaseTestError
 
-    if isinstance(e, ReleaseTestError):
-        exit_code = e.exit_code
-        # Legacy reporting
-        if 1 <= exit_code.value < 10:
-            error_type = "runtime_error"
-            runtime = None
-        elif 10 <= exit_code.value < 20:
-            error_type = "infra_error"
-            runtime = None
-        elif 30 <= exit_code.value < 40:
-            error_type = "infra_timeout"
-            runtime = None
-        elif exit_code == ExitCode.COMMAND_TIMEOUT:
-            error_type = "timeout"
-            runtime = 0
-        elif 40 <= exit_code.value < 50:
-            error_type = "error"
-            runtime = 0
-        else:
-            error_type = "error"
-            runtime = 0
-    else:
-        exit_code = ExitCode.UNKNOWN
-        error_type = "unknown error"
+    if not isinstance(e, ReleaseTestError):
+        return ExitCode.UNKNOWN, ResultStatus.UNKNOWN, 0
+    exit_code = e.exit_code
+    if 1 <= exit_code.value < 10:
+        result_status = ResultStatus.RUNTIME_ERROR
+        runtime = None
+    elif 10 <= exit_code.value < 20:
+        result_status = ResultStatus.INFRA_ERROR
+        runtime = None
+    elif 30 <= exit_code.value < 40:
+        result_status = ResultStatus.INFRA_TIMEOUT
+        runtime = None
+    elif exit_code == ExitCode.COMMAND_TIMEOUT:
+        result_status = ResultStatus.TIMEOUT
         runtime = 0
+    elif 40 <= exit_code.value:
+        result_status = ResultStatus.ERROR
+        runtime = 0
+
+    # if this result is to be retried, mark its status as transient
+    # this logic should be in-sync with run_release_test.sh
+    if result_status in [ResultStatus.INFRA_ERROR, ResultStatus.INFRA_TIMEOUT]:
+        retry_count = int(os.environ.get("BUILDKITE_RETRY_COUNT", 0))
+        max_retry = int(os.environ.get("BUILDKITE_MAX_RETRIES", 1))
+        if retry_count < max_retry:
+            result_status = ResultStatus.TRANSIENT_INFRA_ERROR
 
-    return exit_code, error_type, runtime
+    return exit_code, result_status, runtime

release/ray_release/tests/test_glue.py

@@ -621,7 +621,7 @@ def testFetchResultFailsReqNonEmptyResult(self):
                 self._run(result)
                 self.assertTrue(any("Could not fetch results" in o for o in cm.output))
         self.assertEqual(result.return_code, ExitCode.FETCH_RESULT_ERROR.value)
-        self.assertEqual(result.status, "infra_error")
+        self.assertEqual(result.status, "transient_infra_error")
 
         # Ensure cluster was terminated, no matter what
         self.assertGreaterEqual(self.sdk.call_counter["terminate_cluster"], 1)
@@ -638,7 +638,6 @@ def testLastLogsFails(self):
             self.assertTrue(any("Error fetching logs" in o for o in cm.output))
         self.assertEqual(result.return_code, ExitCode.SUCCESS.value)
         self.assertEqual(result.status, "finished")
-        self.assertIn("No logs", result.last_logs)
 
         # Ensure cluster was terminated
         self.assertGreaterEqual(self.sdk.call_counter["terminate_cluster"], 1)

release/ray_release/tests/test_run_script.py

@@ -94,7 +94,7 @@ def test_repeat(setup):
             ExitCode.CLUSTER_WAIT_TIMEOUT,
             ExitCode.RAY_WHEELS_TIMEOUT,
         )
-        == ExitCode.RAY_WHEELS_TIMEOUT.value
+        == 79  # BUILDKITE_RETRY_CODE
     )
     assert _read_state(state_file) == 3
 

release/run_release_test.sh

@@ -30,13 +30,15 @@ RAY_TEST_SCRIPT=${RAY_TEST_SCRIPT-ray_release/scripts/run_release_test.py}
 RAY_TEST_REPO=${RAY_TEST_REPO-https://github.com/ray-project/ray.git}
 RAY_TEST_BRANCH=${RAY_TEST_BRANCH-master}
 RELEASE_RESULTS_DIR=${RELEASE_RESULTS_DIR-/tmp/artifacts}
+BUILDKITE_MAX_RETRIES=1
+BUILDKITE_RETRY_CODE=79
 
 # This is not a great idea if your OS is different to the one
 # used in the product clusters. However, we need this in CI as reloading
 # Ray within the python process does not work for protobuf changes.
 INSTALL_MATCHING_RAY=${BUILDKITE-false}
 
-export RAY_TEST_REPO RAY_TEST_BRANCH RELEASE_RESULTS_DIR
+export RAY_TEST_REPO RAY_TEST_BRANCH RELEASE_RESULTS_DIR BUILDKITE_MAX_RETRIES BUILDKITE_RETRY_CODE
 
 if [ -z "${NO_INSTALL}" ]; then
   pip install --use-deprecated=legacy-resolver -q -r requirements.txt
@@ -173,4 +175,8 @@ if [ -z "${NO_CLONE}" ]; then
   rm -rf "${TMPDIR}" || true
 fi
 
-exit $EXIT_CODE
+if [ "$REASON" == "infra error" ] || [ "$REASON" == "infra timeout" ]; then
+  exit $BUILDKITE_RETRY_CODE
+else
+  exit $EXIT_CODE
+fi