[ci][config/5] make ci pipelines configurable (ray-project#44432)

Make CI pipelines configurable, they are different upstream.

Signed-off-by: can <[email protected]>

ci/ray_ci/oss_config.yaml

@@ -6,6 +6,13 @@ release_byod:
   aws_cr: 029272617770.dkr.ecr.us-west-2.amazonaws.com
   gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci
   aws2gce_credentials: release/aws2gce_iam.json
+ci_pipeline:
+  premerge:
+    - 0189942e-0876-4b8f-80a4-617f988ec59b  # premerge
+  postmerge:
+    - 0189e759-8c96-4302-b6b5-b4274406bf89  # postmerge
+    - 018e0f94-ccb6-45c2-b072-1e624fe9a404  # postmerge-macos
+    - 018af6d3-58e1-463f-90ec-d9aa4a4f57f1  # release
 state_machine:
   pr:
     aws_bucket: ray-ci-pr-results

ci/ray_ci/ray_docker_container.py

@@ -5,7 +5,7 @@
 from ci.ray_ci.docker_container import DockerContainer
 from ci.ray_ci.builder_container import PYTHON_VERSIONS, DEFAULT_ARCHITECTURE
 from ci.ray_ci.utils import docker_pull, RAY_VERSION
-from ray_release.configs.global_config import BRANCH_PIPELINES
+from ray_release.configs.global_config import get_global_config
 
 
 class RayDockerContainer(DockerContainer):
@@ -59,7 +59,10 @@ def run(self) -> None:
     def _should_upload(self) -> bool:
         if not self.upload:
             return False
-        if os.environ.get("BUILDKITE_PIPELINE_ID") not in BRANCH_PIPELINES:
+        if (
+            os.environ.get("BUILDKITE_PIPELINE_ID")
+            not in get_global_config()["ci_pipeline_postmerge"]
+        ):
             return False
         if os.environ.get("BUILDKITE_BRANCH", "").startswith("releases/"):
             return True

ci/ray_ci/test_base.py

@@ -4,10 +4,12 @@
 
 from ci.ray_ci.builder_container import PYTHON_VERSIONS
 from ci.ray_ci.builder import DEFAULT_PYTHON_VERSION
+from ci.ray_ci.utils import ci_init
 
 
 class RayCITestBase(unittest.TestCase):
     def setUp(self) -> None:
+        ci_init()
         self.patcher = patch.dict(
             os.environ,
             {

ci/ray_ci/test_linux_tester_container.py

@@ -8,9 +8,12 @@
 from typing import List, Optional
 
 from ci.ray_ci.linux_tester_container import LinuxTesterContainer
-from ci.ray_ci.utils import chunk_into_n
+from ci.ray_ci.utils import chunk_into_n, ci_init
 from ci.ray_ci.container import _DOCKER_ECR_REPO, _RAYCI_BUILD_ID
-from ray_release.configs.global_config import BRANCH_PIPELINES, PR_PIPELINES
+from ray_release.configs.global_config import get_global_config
+
+
+ci_init()
 
 
 class MockPopen:
@@ -47,7 +50,7 @@ def test_persist_test_results(
         os.environ,
         {
             "BUILDKITE_BRANCH": "non-master",
-            "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+            "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0],
         },
     ):
         container._persist_test_results("team", "log_dir")
@@ -57,7 +60,7 @@ def test_persist_test_results(
         os.environ,
         {
             "BUILDKITE_BRANCH": "non-master",
-            "BUILDKITE_PIPELINE_ID": PR_PIPELINES[0],
+            "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_premerge"][0],
         },
     ):
         container._persist_test_results("team", "log_dir")
@@ -67,7 +70,7 @@ def test_persist_test_results(
         os.environ,
         {
             "BUILDKITE_BRANCH": "master",
-            "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+            "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0],
         },
     ):
         container._persist_test_results("team", "log_dir")

ci/ray_ci/test_ray_docker_container.py

@@ -10,7 +10,7 @@
 from ci.ray_ci.ray_docker_container import RayDockerContainer
 from ci.ray_ci.test_base import RayCITestBase
 from ci.ray_ci.utils import RAY_VERSION
-from ray_release.configs.global_config import BRANCH_PIPELINES
+from ray_release.configs.global_config import get_global_config
 
 
 class TestRayDockerContainer(RayCITestBase):
@@ -320,14 +320,18 @@ def test_should_upload(self) -> None:
             # environment_variables, expected_result (with upload flag on)
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][0],
                     "BUILDKITE_BRANCH": "releases/1.0.0",
                 },
                 True,  # satisfy upload requirements
             ),
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][0],
                     "BUILDKITE_BRANCH": "master",
                     "RAYCI_SCHEDULE": "nightly",
                 },
@@ -343,14 +347,18 @@ def test_should_upload(self) -> None:
             ),
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[-1],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][-1],
                     "BUILDKITE_BRANCH": "non-release/1.2.3",
                 },
                 False,  # not satisfied: branch is not release/master
             ),
             (
                 {
-                    "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[-1],
+                    "BUILDKITE_PIPELINE_ID": get_global_config()[
+                        "ci_pipeline_postmerge"
+                    ][-1],
                     "BUILDKITE_BRANCH": "123",
                     "RAYCI_SCHEDULE": "nightly",
                 },

ci/ray_ci/tester_container.py

@@ -13,7 +13,7 @@
 from ci.ray_ci.container import Container
 from ray_release.test import TestResult, Test
 from ray_release.test_automation.ci_state_machine import CITestStateMachine
-from ray_release.configs.global_config import BRANCH_PIPELINES, PR_PIPELINES
+from ray_release.configs.global_config import get_global_config
 
 
 class TesterContainer(Container):
@@ -108,13 +108,15 @@ def run_tests(
     def _persist_test_results(self, team: str, bazel_log_dir: str) -> None:
         pipeline_id = os.environ.get("BUILDKITE_PIPELINE_ID")
         branch = os.environ.get("BUILDKITE_BRANCH")
-        if pipeline_id not in BRANCH_PIPELINES + PR_PIPELINES:
+        branch_pipelines = get_global_config()["ci_pipeline_postmerge"]
+        pr_pipelines = get_global_config()["ci_pipeline_premerge"]
+        if pipeline_id not in branch_pipelines + pr_pipelines:
             logger.info(
                 "Skip upload test results. "
                 "We only upload results on branch and PR pipelines",
             )
             return
-        if pipeline_id in BRANCH_PIPELINES and branch != "master":
+        if pipeline_id in branch_pipelines and branch != "master":
             logger.info(
                 "Skip upload test results. "
                 "We only upload the master branch results on a branch pipeline",
@@ -146,7 +148,10 @@ def upload_test_results(cls, team: str, bazel_log_dir: str) -> None:
     def move_test_state(cls, team: str, bazel_log_dir: str) -> None:
         pipeline_id = os.environ.get("BUILDKITE_PIPELINE_ID")
         branch = os.environ.get("BUILDKITE_BRANCH")
-        if pipeline_id not in BRANCH_PIPELINES or branch != "master":
+        if (
+            pipeline_id not in get_global_config()["ci_pipeline_postmerge"]
+            or branch != "master"
+        ):
             logger.info("Skip updating test state. We only update on master branch.")
             return
         for test, _ in cls.get_test_and_results(team, bazel_log_dir):

release/ray_release/configs/global_config.py

@@ -1,17 +1,9 @@
 import os
 
 import yaml
+from typing import List
 from typing_extensions import TypedDict
 
-PR_PIPELINES = [
-    "0189942e-0876-4b8f-80a4-617f988ec59b",  # premerge
-]
-BRANCH_PIPELINES = [
-    "0189e759-8c96-4302-b6b5-b4274406bf89",  # postmerge
-    "018e0f94-ccb6-45c2-b072-1e624fe9a404",  # postmerge-macos
-    "018af6d3-58e1-463f-90ec-d9aa4a4f57f1",  # release
-]
-
 
 class GlobalConfig(TypedDict):
     byod_ray_ecr: str
@@ -24,6 +16,8 @@ class GlobalConfig(TypedDict):
     state_machine_pr_aws_bucket: str
     state_machine_branch_aws_bucket: str
     aws2gce_credentials: str
+    ci_pipeline_premerge: List[str]
+    ci_pipeline_postmerge: List[str]
 
 
 config = None
@@ -92,6 +86,10 @@ def _init_global_config(config_file: str):
         .get(
             "aws_bucket",
         ),
+        ci_pipeline_premerge=config_content.get("ci_pipeline", {}).get("premerge", []),
+        ci_pipeline_postmerge=config_content.get("ci_pipeline", {}).get(
+            "postmerge", []
+        ),
     )
     # setup GCP workload identity federation
     os.environ[

release/ray_release/tests/test_global_config.py

@@ -22,6 +22,12 @@
   aws_bucket: ray-ci-results
 credentials:
   aws2gce: release/aws2gce_iam.json
+ci_pipeline:
+  premerge:
+    - w00t
+  postmerge:
+    - hi
+    - three
 """
 
 
@@ -34,6 +40,8 @@ def test_init_global_config() -> None:
         config = get_global_config()
         assert config["byod_ray_ecr"] == "rayproject"
         assert config["aws2gce_credentials"] == "release/aws2gce_iam.json"
+        assert config["ci_pipeline_premerge"] == ["w00t"]
+        assert config["ci_pipeline_postmerge"] == ["hi", "three"]
         assert (
             os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
             == "/workdir/release/aws2gce_iam.json"

release/ray_release/tests/test_test.py

@@ -12,7 +12,6 @@
 from ray_release.configs.global_config import (
     init_global_config,
     get_global_config,
-    BRANCH_PIPELINES,
 )
 from ray_release.test import (
     Test,
@@ -190,7 +189,10 @@ def test_from_bazel_event() -> None:
 
 
 @patch.object(boto3, "client")
-@patch.dict(os.environ, {"BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0]})
+@patch.dict(
+    os.environ,
+    {"BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0]},
+)
 def test_update_from_s3(mock_client) -> None:
     mock_object = mock.Mock()
     mock_object.return_value.get.return_value.read.return_value = json.dumps(

release/ray_release/util.py

@@ -10,11 +10,7 @@
 
 import requests
 from ray_release.logger import logger
-from ray_release.configs.global_config import (
-    get_global_config,
-    PR_PIPELINES,
-    BRANCH_PIPELINES,
-)
+from ray_release.configs.global_config import get_global_config
 
 if TYPE_CHECKING:
     from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
@@ -60,11 +56,13 @@ def get_write_state_machine_aws_bucket() -> str:
         return bucket_v1
 
     pipeline_id = os.environ.get("BUILDKITE_PIPELINE_ID")
-    assert pipeline_id in BRANCH_PIPELINES + PR_PIPELINES, (
+    pr_pipelines = get_global_config()["ci_pipeline_premerge"]
+    branch_pipelines = get_global_config()["ci_pipeline_postmerge"]
+    assert pipeline_id in pr_pipelines + branch_pipelines, (
         "Test state machine is only supported for branch or pr pipeline, "
         f"{pipeline_id} is given"
     )
-    if pipeline_id in PR_PIPELINES:
+    if pipeline_id in pr_pipelines:
         return get_global_config()["state_machine_pr_aws_bucket"]
     return get_global_config()["state_machine_branch_aws_bucket"]