Skip to content

Commit

Permalink
[ci][bisect/4] move bisect logic to state machine base class (ray-pro…
Browse files Browse the repository at this point in the history
…ject#44807)

Move bisect logic to state machine base class. Purely moving things over. This is to prepare for reusing this logic for macos tests.

---------

Signed-off-by: can <[email protected]>
  • Loading branch information
can-anyscale authored Apr 24, 2024
1 parent 24539f8 commit a1c3619
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 55 deletions.
54 changes: 0 additions & 54 deletions release/ray_release/test_automation/release_state_machine.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,10 @@
from datetime import datetime, timedelta

from ray_release.test_automation.state_machine import (
TestStateMachine,
WEEKLY_RELEASE_BLOCKER_TAG,
BUILDKITE_BISECT_PIPELINE,
BUILDKITE_ORGANIZATION,
)
from ray_release.test import Test, TestState
from ray_release.logger import logger


MAX_BISECT_PER_DAY = 10 # Max number of bisects to run per day for all tests
CONTINUOUS_FAILURE_TO_JAIL = 3 # Number of continuous failures before jailing
UNSTABLE_RELEASE_TEST_TAG = "unstable-release-test"

Expand Down Expand Up @@ -88,51 +82,3 @@ def _create_github_issue(self) -> None:
labels=labels,
).number
self.test[Test.KEY_GITHUB_ISSUE_NUMBER] = issue_number

def _bisect_rate_limit_exceeded(self) -> bool:
"""
Check if we have exceeded the rate limit of bisects per day.
"""
builds = self.ray_buildkite.builds().list_all_for_pipeline(
BUILDKITE_ORGANIZATION,
BUILDKITE_BISECT_PIPELINE,
created_from=datetime.now() - timedelta(days=1),
branch="master",
)
return len(builds) >= MAX_BISECT_PER_DAY

def _trigger_bisect(self) -> None:
if self._bisect_rate_limit_exceeded():
logger.info(f"Skip bisect {self.test.get_name()} due to rate limit")
return
build = self.ray_buildkite.builds().create_build(
BUILDKITE_ORGANIZATION,
BUILDKITE_BISECT_PIPELINE,
"HEAD",
"master",
message=f"[ray-test-bot] {self.test.get_name()} failing",
env={
"UPDATE_TEST_STATE_MACHINE": "1",
},
)
failing_commit = self.test_results[0].commit
passing_commits = [r.commit for r in self.test_results if r.is_passing()]
if not passing_commits:
logger.info(f"Skip bisect {self.test.get_name()} due to no passing commit")
return
passing_commit = passing_commits[0]
self.ray_buildkite.jobs().unblock_job(
BUILDKITE_ORGANIZATION,
BUILDKITE_BISECT_PIPELINE,
build["number"],
build["jobs"][0]["id"], # first job is the blocked job
fields={
"test-name": self.test.get_name(),
"passing-commit": passing_commit,
"failing-commit": failing_commit,
"concurrency": "3",
"run-per-commit": "1",
"test-type": "release-test",
},
)
self.test[Test.KEY_BISECT_BUILD_NUMBER] = build["number"]
51 changes: 50 additions & 1 deletion release/ray_release/test_automation/state_machine.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import abc
from typing import List
from datetime import datetime, timedelta

import github
from github import Github
Expand Down Expand Up @@ -28,6 +29,7 @@
"serve",
"serverless",
]
MAX_BISECT_PER_DAY = 10 # Max number of bisects to run per day for all tests


class TestStateMachine(abc.ABC):
Expand Down Expand Up @@ -276,5 +278,52 @@ def comment_blamed_commit_on_github_issue(self) -> bool:
f"found by bisect job https://buildkite.com/{BUILDKITE_ORGANIZATION}/"
f"{BUILDKITE_BISECT_PIPELINE}/builds/{bisect_build_number}"
)

return True

def _trigger_bisect(self) -> None:
if self._bisect_rate_limit_exceeded():
logger.info(f"Skip bisect {self.test.get_name()} due to rate limit")
return
build = self.ray_buildkite.builds().create_build(
BUILDKITE_ORGANIZATION,
BUILDKITE_BISECT_PIPELINE,
"HEAD",
"master",
message=f"[ray-test-bot] {self.test.get_name()} failing",
env={
"UPDATE_TEST_STATE_MACHINE": "1",
},
)
failing_commit = self.test_results[0].commit
passing_commits = [r.commit for r in self.test_results if r.is_passing()]
if not passing_commits:
logger.info(f"Skip bisect {self.test.get_name()} due to no passing commit")
return
passing_commit = passing_commits[0]
self.ray_buildkite.jobs().unblock_job(
BUILDKITE_ORGANIZATION,
BUILDKITE_BISECT_PIPELINE,
build["number"],
build["jobs"][0]["id"], # first job is the blocked job
fields={
"test-name": self.test.get_name(),
"passing-commit": passing_commit,
"failing-commit": failing_commit,
"concurrency": "3",
"run-per-commit": "1",
"test-type": "release-test",
},
)
self.test[Test.KEY_BISECT_BUILD_NUMBER] = build["number"]

def _bisect_rate_limit_exceeded(self) -> bool:
"""
Check if we have exceeded the rate limit of bisects per day.
"""
builds = self.ray_buildkite.builds().list_all_for_pipeline(
BUILDKITE_ORGANIZATION,
BUILDKITE_BISECT_PIPELINE,
created_from=datetime.now() - timedelta(days=1),
branch="master",
)
return len(builds) >= MAX_BISECT_PER_DAY

0 comments on commit a1c3619

Please sign in to comment.