From 2ebab1c1578320820f3c86114713978d13f3f286 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:55:44 -0700 Subject: [PATCH] [ci][config/5] make ci pipelines configurable (#44432) Make CI pipelines configurable, they are different upstream. Signed-off-by: can --- ci/ray_ci/oss_config.yaml | 7 +++++++ ci/ray_ci/ray_docker_container.py | 7 +++++-- ci/ray_ci/test_base.py | 2 ++ ci/ray_ci/test_linux_tester_container.py | 13 ++++++++----- ci/ray_ci/test_ray_docker_container.py | 18 +++++++++++++----- ci/ray_ci/tester_container.py | 13 +++++++++---- release/ray_release/configs/global_config.py | 16 +++++++--------- .../ray_release/tests/test_global_config.py | 8 ++++++++ release/ray_release/tests/test_test.py | 6 ++++-- release/ray_release/util.py | 12 +++++------- 10 files changed, 68 insertions(+), 34 deletions(-) diff --git a/ci/ray_ci/oss_config.yaml b/ci/ray_ci/oss_config.yaml index 7ffcaa866ef6..de69060ee1c1 100644 --- a/ci/ray_ci/oss_config.yaml +++ b/ci/ray_ci/oss_config.yaml @@ -6,6 +6,13 @@ release_byod: aws_cr: 029272617770.dkr.ecr.us-west-2.amazonaws.com gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci aws2gce_credentials: release/aws2gce_iam.json +ci_pipeline: + premerge: + - 0189942e-0876-4b8f-80a4-617f988ec59b # premerge + postmerge: + - 0189e759-8c96-4302-b6b5-b4274406bf89 # postmerge + - 018e0f94-ccb6-45c2-b072-1e624fe9a404 # postmerge-macos + - 018af6d3-58e1-463f-90ec-d9aa4a4f57f1 # release state_machine: pr: aws_bucket: ray-ci-pr-results diff --git a/ci/ray_ci/ray_docker_container.py b/ci/ray_ci/ray_docker_container.py index 73ebca2321fb..1e876bbf5d86 100644 --- a/ci/ray_ci/ray_docker_container.py +++ b/ci/ray_ci/ray_docker_container.py @@ -5,7 +5,7 @@ from ci.ray_ci.docker_container import DockerContainer from ci.ray_ci.builder_container import PYTHON_VERSIONS, DEFAULT_ARCHITECTURE from ci.ray_ci.utils import docker_pull, RAY_VERSION -from ray_release.configs.global_config import BRANCH_PIPELINES +from ray_release.configs.global_config import get_global_config class RayDockerContainer(DockerContainer): @@ -59,7 +59,10 @@ def run(self) -> None: def _should_upload(self) -> bool: if not self.upload: return False - if os.environ.get("BUILDKITE_PIPELINE_ID") not in BRANCH_PIPELINES: + if ( + os.environ.get("BUILDKITE_PIPELINE_ID") + not in get_global_config()["ci_pipeline_postmerge"] + ): return False if os.environ.get("BUILDKITE_BRANCH", "").startswith("releases/"): return True diff --git a/ci/ray_ci/test_base.py b/ci/ray_ci/test_base.py index a690c542a681..e5c5d0b76679 100644 --- a/ci/ray_ci/test_base.py +++ b/ci/ray_ci/test_base.py @@ -4,10 +4,12 @@ from ci.ray_ci.builder_container import PYTHON_VERSIONS from ci.ray_ci.builder import DEFAULT_PYTHON_VERSION +from ci.ray_ci.utils import ci_init class RayCITestBase(unittest.TestCase): def setUp(self) -> None: + ci_init() self.patcher = patch.dict( os.environ, { diff --git a/ci/ray_ci/test_linux_tester_container.py b/ci/ray_ci/test_linux_tester_container.py index e59fb83f14d6..fd5c308c4eeb 100644 --- a/ci/ray_ci/test_linux_tester_container.py +++ b/ci/ray_ci/test_linux_tester_container.py @@ -8,9 +8,12 @@ from typing import List, Optional from ci.ray_ci.linux_tester_container import LinuxTesterContainer -from ci.ray_ci.utils import chunk_into_n +from ci.ray_ci.utils import chunk_into_n, ci_init from ci.ray_ci.container import _DOCKER_ECR_REPO, _RAYCI_BUILD_ID -from ray_release.configs.global_config import BRANCH_PIPELINES, PR_PIPELINES +from ray_release.configs.global_config import get_global_config + + +ci_init() class MockPopen: @@ -47,7 +50,7 @@ def test_persist_test_results( os.environ, { "BUILDKITE_BRANCH": "non-master", - "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0], + "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0], }, ): container._persist_test_results("team", "log_dir") @@ -57,7 +60,7 @@ def test_persist_test_results( os.environ, { "BUILDKITE_BRANCH": "non-master", - "BUILDKITE_PIPELINE_ID": PR_PIPELINES[0], + "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_premerge"][0], }, ): container._persist_test_results("team", "log_dir") @@ -67,7 +70,7 @@ def test_persist_test_results( os.environ, { "BUILDKITE_BRANCH": "master", - "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0], + "BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0], }, ): container._persist_test_results("team", "log_dir") diff --git a/ci/ray_ci/test_ray_docker_container.py b/ci/ray_ci/test_ray_docker_container.py index ac430c9b71f0..0b78f258bfa6 100644 --- a/ci/ray_ci/test_ray_docker_container.py +++ b/ci/ray_ci/test_ray_docker_container.py @@ -10,7 +10,7 @@ from ci.ray_ci.ray_docker_container import RayDockerContainer from ci.ray_ci.test_base import RayCITestBase from ci.ray_ci.utils import RAY_VERSION -from ray_release.configs.global_config import BRANCH_PIPELINES +from ray_release.configs.global_config import get_global_config class TestRayDockerContainer(RayCITestBase): @@ -320,14 +320,18 @@ def test_should_upload(self) -> None: # environment_variables, expected_result (with upload flag on) ( { - "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0], + "BUILDKITE_PIPELINE_ID": get_global_config()[ + "ci_pipeline_postmerge" + ][0], "BUILDKITE_BRANCH": "releases/1.0.0", }, True, # satisfy upload requirements ), ( { - "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0], + "BUILDKITE_PIPELINE_ID": get_global_config()[ + "ci_pipeline_postmerge" + ][0], "BUILDKITE_BRANCH": "master", "RAYCI_SCHEDULE": "nightly", }, @@ -343,14 +347,18 @@ def test_should_upload(self) -> None: ), ( { - "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[-1], + "BUILDKITE_PIPELINE_ID": get_global_config()[ + "ci_pipeline_postmerge" + ][-1], "BUILDKITE_BRANCH": "non-release/1.2.3", }, False, # not satisfied: branch is not release/master ), ( { - "BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[-1], + "BUILDKITE_PIPELINE_ID": get_global_config()[ + "ci_pipeline_postmerge" + ][-1], "BUILDKITE_BRANCH": "123", "RAYCI_SCHEDULE": "nightly", }, diff --git a/ci/ray_ci/tester_container.py b/ci/ray_ci/tester_container.py index 8ddbe0ef0983..b940cac763c1 100644 --- a/ci/ray_ci/tester_container.py +++ b/ci/ray_ci/tester_container.py @@ -13,7 +13,7 @@ from ci.ray_ci.container import Container from ray_release.test import TestResult, Test from ray_release.test_automation.ci_state_machine import CITestStateMachine -from ray_release.configs.global_config import BRANCH_PIPELINES, PR_PIPELINES +from ray_release.configs.global_config import get_global_config class TesterContainer(Container): @@ -108,13 +108,15 @@ def run_tests( def _persist_test_results(self, team: str, bazel_log_dir: str) -> None: pipeline_id = os.environ.get("BUILDKITE_PIPELINE_ID") branch = os.environ.get("BUILDKITE_BRANCH") - if pipeline_id not in BRANCH_PIPELINES + PR_PIPELINES: + branch_pipelines = get_global_config()["ci_pipeline_postmerge"] + pr_pipelines = get_global_config()["ci_pipeline_premerge"] + if pipeline_id not in branch_pipelines + pr_pipelines: logger.info( "Skip upload test results. " "We only upload results on branch and PR pipelines", ) return - if pipeline_id in BRANCH_PIPELINES and branch != "master": + if pipeline_id in branch_pipelines and branch != "master": logger.info( "Skip upload test results. " "We only upload the master branch results on a branch pipeline", @@ -146,7 +148,10 @@ def upload_test_results(cls, team: str, bazel_log_dir: str) -> None: def move_test_state(cls, team: str, bazel_log_dir: str) -> None: pipeline_id = os.environ.get("BUILDKITE_PIPELINE_ID") branch = os.environ.get("BUILDKITE_BRANCH") - if pipeline_id not in BRANCH_PIPELINES or branch != "master": + if ( + pipeline_id not in get_global_config()["ci_pipeline_postmerge"] + or branch != "master" + ): logger.info("Skip updating test state. We only update on master branch.") return for test, _ in cls.get_test_and_results(team, bazel_log_dir): diff --git a/release/ray_release/configs/global_config.py b/release/ray_release/configs/global_config.py index 803c05d3b6e1..265c4d70c751 100644 --- a/release/ray_release/configs/global_config.py +++ b/release/ray_release/configs/global_config.py @@ -1,17 +1,9 @@ import os import yaml +from typing import List from typing_extensions import TypedDict -PR_PIPELINES = [ - "0189942e-0876-4b8f-80a4-617f988ec59b", # premerge -] -BRANCH_PIPELINES = [ - "0189e759-8c96-4302-b6b5-b4274406bf89", # postmerge - "018e0f94-ccb6-45c2-b072-1e624fe9a404", # postmerge-macos - "018af6d3-58e1-463f-90ec-d9aa4a4f57f1", # release -] - class GlobalConfig(TypedDict): byod_ray_ecr: str @@ -24,6 +16,8 @@ class GlobalConfig(TypedDict): state_machine_pr_aws_bucket: str state_machine_branch_aws_bucket: str aws2gce_credentials: str + ci_pipeline_premerge: List[str] + ci_pipeline_postmerge: List[str] config = None @@ -92,6 +86,10 @@ def _init_global_config(config_file: str): .get( "aws_bucket", ), + ci_pipeline_premerge=config_content.get("ci_pipeline", {}).get("premerge", []), + ci_pipeline_postmerge=config_content.get("ci_pipeline", {}).get( + "postmerge", [] + ), ) # setup GCP workload identity federation os.environ[ diff --git a/release/ray_release/tests/test_global_config.py b/release/ray_release/tests/test_global_config.py index e746cad721a3..5864dc4a997d 100644 --- a/release/ray_release/tests/test_global_config.py +++ b/release/ray_release/tests/test_global_config.py @@ -22,6 +22,12 @@ aws_bucket: ray-ci-results credentials: aws2gce: release/aws2gce_iam.json +ci_pipeline: + premerge: + - w00t + postmerge: + - hi + - three """ @@ -34,6 +40,8 @@ def test_init_global_config() -> None: config = get_global_config() assert config["byod_ray_ecr"] == "rayproject" assert config["aws2gce_credentials"] == "release/aws2gce_iam.json" + assert config["ci_pipeline_premerge"] == ["w00t"] + assert config["ci_pipeline_postmerge"] == ["hi", "three"] assert ( os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == "/workdir/release/aws2gce_iam.json" diff --git a/release/ray_release/tests/test_test.py b/release/ray_release/tests/test_test.py index f0f03eab7c59..8c36d31545c1 100644 --- a/release/ray_release/tests/test_test.py +++ b/release/ray_release/tests/test_test.py @@ -12,7 +12,6 @@ from ray_release.configs.global_config import ( init_global_config, get_global_config, - BRANCH_PIPELINES, ) from ray_release.test import ( Test, @@ -190,7 +189,10 @@ def test_from_bazel_event() -> None: @patch.object(boto3, "client") -@patch.dict(os.environ, {"BUILDKITE_PIPELINE_ID": BRANCH_PIPELINES[0]}) +@patch.dict( + os.environ, + {"BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0]}, +) def test_update_from_s3(mock_client) -> None: mock_object = mock.Mock() mock_object.return_value.get.return_value.read.return_value = json.dumps( diff --git a/release/ray_release/util.py b/release/ray_release/util.py index 4938e09464cc..aa390514e53b 100644 --- a/release/ray_release/util.py +++ b/release/ray_release/util.py @@ -10,11 +10,7 @@ import requests from ray_release.logger import logger -from ray_release.configs.global_config import ( - get_global_config, - PR_PIPELINES, - BRANCH_PIPELINES, -) +from ray_release.configs.global_config import get_global_config if TYPE_CHECKING: from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK @@ -60,11 +56,13 @@ def get_write_state_machine_aws_bucket() -> str: return bucket_v1 pipeline_id = os.environ.get("BUILDKITE_PIPELINE_ID") - assert pipeline_id in BRANCH_PIPELINES + PR_PIPELINES, ( + pr_pipelines = get_global_config()["ci_pipeline_premerge"] + branch_pipelines = get_global_config()["ci_pipeline_postmerge"] + assert pipeline_id in pr_pipelines + branch_pipelines, ( "Test state machine is only supported for branch or pr pipeline, " f"{pipeline_id} is given" ) - if pipeline_id in PR_PIPELINES: + if pipeline_id in pr_pipelines: return get_global_config()["state_machine_pr_aws_bucket"] return get_global_config()["state_machine_branch_aws_bucket"]