[ci][bisect/1] initial version of macos test bisect (ray-project#44618)

An initial version of macos test bisect.

- A base class for bisecting
- A macos test bisect which overrides the logic of how to run a single test
Bisecting a single test on macos takes 40 minutes and one machine. It can be optimized to be faster.

Signed-off-by: can <[email protected]>

ci/ray_ci/bisect/BUILD.bazel

@@ -0,0 +1,48 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_binary(
+    name = "bisect_test",
+    srcs = ["bisect_test.py"],
+    exec_compatible_with = ["//:hermetic_python"],
+    deps = [":bisect"],
+    data = [":macos_validator"],
+)
+
+genrule(
+    name = "macos_validator",
+    srcs = ["macos_ci.sh"],
+    outs = ["macos_validator.sh"],
+    cmd = """
+        cat $(location macos_ci.sh) > $@
+    """,
+)
+
+py_library(
+    name = "bisect",
+    srcs = glob(
+        ["*.py"],
+        exclude = [
+            "test_*.py",
+        ],
+    ),
+    visibility = ["//ci/ray_ci/bisect:__subpackages__"],
+    deps = [
+        "//ci/ray_ci:ray_ci_lib",
+    ],
+)
+
+py_test(
+    name = "test_bisector",
+    size = "small",
+    srcs = ["test_bisector.py"],
+    exec_compatible_with = ["//:hermetic_python"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ":bisect",
+        ci_require("pytest"),
+    ],
+)

ci/ray_ci/bisect/bisect_test.py

@@ -0,0 +1,19 @@
+import click
+
+from ci.ray_ci.utils import logger, ci_init
+from ci.ray_ci.bisect.macos_validator import MacOSValidator
+from ci.ray_ci.bisect.bisector import Bisector
+
+
+@click.command()
+@click.argument("test_name", required=True, type=str)
+@click.argument("passing_commit", required=True, type=str)
+@click.argument("failing_commit", required=True, type=str)
+def main(test_name: str, passing_commit: str, failing_commit: str) -> None:
+    ci_init()
+    blame = Bisector(test_name, passing_commit, failing_commit, MacOSValidator()).run()
+    logger.info(f"Blame revision: {blame}")
+
+
+if __name__ == "__main__":
+    main()

ci/ray_ci/bisect/bisector.py

@@ -0,0 +1,70 @@
+import os
+import subprocess
+from typing import List, Optional
+
+from ci.ray_ci.utils import logger
+from ci.ray_ci.bisect.validator import Validator
+
+
+class Bisector:
+    def __init__(
+        self,
+        test: str,
+        passing_revision: str,
+        failing_revision: str,
+        validator: Validator,
+    ) -> None:
+        self.test = test
+        self.passing_revision = passing_revision
+        self.failing_revision = failing_revision
+        self.validator = validator
+
+    def run(self) -> Optional[str]:
+        """
+        Find the blame revision for the test given the range of passing and failing
+        revision. If a blame cannot be found, return None
+        """
+        revisions = self._get_revision_lists()
+        if len(revisions) < 2:
+            return None
+        while len(revisions) > 2:
+            logger.info(
+                f"Bisecting between {len(revisions)} revisions: "
+                f"{revisions[0]} to {revisions[-1]}"
+            )
+            mid = len(revisions) // 2
+            if self._checkout_and_validate(revisions[mid]):
+                revisions = revisions[mid:]
+            else:
+                revisions = revisions[:mid]
+
+        return revisions[-1]
+
+    def _get_revision_lists(self) -> List[str]:
+        return (
+            subprocess.check_output(
+                [
+                    "git",
+                    "rev-list",
+                    "--reverse",
+                    f"^{self.passing_revision}~",
+                    self.failing_revision,
+                ],
+                cwd=os.environ["RAYCI_CHECKOUT_DIR"],
+            )
+            .decode("utf-8")
+            .strip()
+            .split("\n")
+        )
+
+    def _checkout_and_validate(self, revision: str) -> bool:
+        """
+        Validate whether the test is passing or failing on the given revision
+        """
+        subprocess.check_call(
+            ["git", "clean", "-df"], cwd=os.environ["RAYCI_CHECKOUT_DIR"]
+        )
+        subprocess.check_call(
+            ["git", "checkout", revision], cwd=os.environ["RAYCI_CHECKOUT_DIR"]
+        )
+        self.validator.run(self.test)

ci/ray_ci/bisect/macos_ci.sh

@@ -0,0 +1 @@
+../macos/macos_ci.sh

ci/ray_ci/bisect/macos_validator.py

@@ -0,0 +1,19 @@
+import os
+import subprocess
+
+from ci.ray_ci.bisect.validator import Validator
+from ray_release.bazel import bazel_runfile
+
+
+TEST_SCRIPT = "ci/ray_ci/bisect/macos_validator.sh"
+
+
+class MacOSValidator(Validator):
+    def run(self, test: str) -> bool:
+        return (
+            subprocess.run(
+                [f"{bazel_runfile(TEST_SCRIPT)}", "run_tests", test],
+                cwd=os.environ["RAYCI_CHECKOUT_DIR"],
+            ).returncode
+            == 0
+        )

ci/ray_ci/bisect/test_bisector.py

@@ -0,0 +1,26 @@
+import sys
+import pytest
+from unittest import mock
+
+from ci.ray_ci.bisect.bisector import Bisector
+from ci.ray_ci.bisect.macos_validator import MacOSValidator
+
+
+@mock.patch("ci.ray_ci.bisect.bisector.Bisector._checkout_and_validate")
+@mock.patch("ci.ray_ci.bisect.bisector.Bisector._get_revision_lists")
+def test_run(mock_get_revision_lists, mock_checkout_and_validate):
+    def _mock_checkout_and_validate(revision):
+        return True if revision in ["1", "2", "3"] else False
+
+    mock_checkout_and_validate.side_effect = _mock_checkout_and_validate
+    mock_get_revision_lists.return_value = ["1", "2", "3", "4", "5"]
+
+    # Test case 1: P P P F F
+    assert Bisector("test", "1", "5", MacOSValidator()).run() == "3"
+
+    # Test case 2: P F
+    assert Bisector("test", "3", "4", MacOSValidator()).run() == "3"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

ci/ray_ci/bisect/validator.py

@@ -0,0 +1,10 @@
+import abc
+
+
+class Validator(abc.ABC):
+    @abc.abstractmethod
+    def run(self, test: str) -> bool:
+        """
+        Validate whether the test is passing or failing on the given revision
+        """
+        pass