[release] Make sure that test code matches the installed wheel. (ray-…

…project#30156)

Signed-off-by: xwjiang2010 <[email protected]>

release/ray_release/scripts/build_pipeline.py

@@ -22,6 +22,7 @@
     find_and_wait_for_ray_wheels_url,
     find_ray_wheels_url,
     get_buildkite_repo_branch,
+    parse_commit_from_wheel_url,
 )
 
 PIPELINE_ARTIFACT_PATH = "/tmp/pipeline_artifacts"
@@ -185,6 +186,9 @@ def main(test_collection_file: Optional[str] = None, no_clone_repo: bool = False
             else:
                 this_ray_wheels_url = ray_wheels_url
 
+            ray_commit = parse_commit_from_wheel_url(this_ray_wheels_url)
+            if ray_commit:
+                env.update({"RAY_COMMIT_OF_WHEEL": ray_commit})
             step = get_step(
                 test,
                 report=report,

release/ray_release/tests/test_wheels.py

@@ -19,6 +19,7 @@
     is_wheels_url_matching_ray_verison,
     get_wheels_filename,
     maybe_rewrite_wheels_url,
+    parse_commit_from_wheel_url,
 )
 
 
@@ -252,5 +253,15 @@ def test_url_exist():
     assert not url_exists("invalid://somewhere")
 
 
+def test_parse_commit_from_wheel_url():
+    url = (
+        "https://s3-us-west-2.amazonaws.com/ray-wheels/master/"
+        "0e0c15065507f01e8bfe78e49b0d0de063f81164/"
+        "ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
+    )
+    expected_commit = "0e0c15065507f01e8bfe78e49b0d0de063f81164"
+    assert parse_commit_from_wheel_url(url) == expected_commit
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main(["-v", __file__]))

release/ray_release/wheels.py

@@ -437,3 +437,12 @@ def install_matching_ray_locally(ray_wheels: Optional[str]):
     for module_name in RELOAD_MODULES:
         if module_name in sys.modules:
             importlib.reload(sys.modules[module_name])
+
+
+def parse_commit_from_wheel_url(url: str) -> str:
+    # url is expected to be in the format of
+    # https://s3-us-west-2.amazonaws.com/ray-wheels/master/0e0c15065507f01e8bfe78e49b0d0de063f81164/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl  # noqa
+    regex = r"/([0-9a-f]{40})/"
+    match = re.search(regex, url)
+    if match:
+        return match.group(1)

release/run_release_test.sh

@@ -69,8 +69,25 @@ fi
 if [ -z "${NO_CLONE}" ]; then
   TMPDIR=$(mktemp -d -t release-XXXXXXXXXX)
   echo "Cloning test repo ${RAY_TEST_REPO} branch ${RAY_TEST_BRANCH}"
-  git clone --depth 1 -b "${RAY_TEST_BRANCH}" "${RAY_TEST_REPO}" "${TMPDIR}"
+  git clone -b "${RAY_TEST_BRANCH}" "${RAY_TEST_REPO}" "${TMPDIR}"
   pushd "${TMPDIR}/release" || true
+  HEAD_COMMIT=$(git rev-parse HEAD)
+  echo "The cloned test repo has head commit of ${HEAD_COMMIT}"
+
+  # We only do this if RAY_TEST_REPO and RAY_TEST_BRANCH are pointing to ray master.
+  # Theoretically, release manager may also run into this issue when manually triggering
+  # release test runs. But cherry-picks are rare and thus it's less likely to run into
+  # this racing condition, ignoring for now.
+  if [ "${RAY_TEST_REPO}" == "https://github.com/ray-project/ray.git" ] && \
+  [[ "${PARSED_RAY_WHEELS}" == *"master"*  ]] && \
+  [ "${RAY_TEST_BRANCH-}" == "master" ] && [ -n "${RAY_COMMIT_OF_WHEEL-}" ] && \
+  [ "${HEAD_COMMIT}" != "${RAY_COMMIT_OF_WHEEL}" ]; then
+    echo "The checked out test code doesn't match with the installed wheel. \
+This is likely due to a racing condition when a PR is landed between \
+a wheel is installed and test code is checked out."
+    echo "Hard resetting from ${HEAD_COMMIT} to ${RAY_COMMIT_OF_WHEEL}."
+    git reset --hard "${RAY_COMMIT_OF_WHEEL}"
+  fi
 fi
 
 if [ -z "${NO_INSTALL}" ]; then