[win] migrate windows tests to civ2 (ray-project#41114)

Signed-off-by: Lonnie Liu <[email protected]>
Signed-off-by: Lonnie Liu <[email protected]>

.buildkite/build.rayci.yml

@@ -46,6 +46,24 @@ steps:
       - forge
     job_env: forge
 
+  - label: ":windows: wheel {{matrix}}"
+    # we have linux/mac wheels tag, but do not have a windows wheels tag..
+    # so using python and core_cpp here, to make sure at least it builds
+    tags:
+      - oss
+      - python
+      - core_cpp
+    job_env: WINDOWS
+    instance_type: windows
+    commands:
+      - export BUILD_ONE_PYTHON_ONLY={{matrix}}
+      - bash /c/workdir/.buildkite/windows_ci.sh wheels
+    matrix:
+      - "3.8"
+      - "3.9"
+      - "3.10"
+      - "3.11"
+
   - label: ":tapioca: build: jar"
     tags:
       - java

.buildkite/core.rayci.yml

@@ -253,7 +253,7 @@ steps:
     job_env: forge
 
   - label: ":ray: core: flaky tests"
-    tags: 
+    tags:
       - python
       - skip-on-premerge
     instance_type: large
@@ -271,3 +271,20 @@ steps:
         --skip-ray-installation
     depends_on: corebuild
     job_env: forge
+
+  - label: ":windows: python tests"
+    tags:
+      - python
+    job_env: WINDOWS
+    instance_type: windows
+    commands:
+      - bash /c/workdir/.buildkite/windows_ci.sh test_python
+    parallelism: 10
+
+  - label: ":windows: core tests"
+    tags:
+      - core_cpp
+    job_env: WINDOWS
+    instance_type: windows
+    commands:
+      - bash /c/workdir/.buildkite/windows_ci.sh test_core

.buildkite/ml.rayci.yml

@@ -223,8 +223,7 @@ steps:
     tags:
       - train
       - branch
-    # run only on postmerge
-    if: build.env("BUILDKITE_PIPELINE_ID") == "0189e759-8c96-4302-b6b5-b4274406bf89"
+      - skip-on-premerge
     instance_type: medium
     commands:
       - pip install -U boto3==1.28.70 awscli==1.29.70 
@@ -250,3 +249,11 @@ steps:
     depends_on: mlgpubuild
     job_env: forge
     soft_fail: true
+
+  - label: ":windows: train tests"
+    tags:
+      - train
+    job_env: WINDOWS
+    instance_type: windows
+    commands:
+      - bash /c/workdir/.buildkite/windows_ci.sh test_train_windows

.buildkite/serve.rayci.yml

@@ -187,3 +187,12 @@ steps:
     env:
       PYTHON_VERSION: "3.8"
       EXTRA_DEPENDENCY: "{{matrix}}"
+
+  - label: ":windows: serve tests"
+    tags:
+      - serve
+    job_env: WINDOWS
+    instance_type: windows
+    parallelism: 5
+    commands:
+      - bash /c/workdir/.buildkite/windows_ci.sh test_serve

.buildkite/windows_ci.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -ex
+
+# We already checked out once at /c/workdir, but needs to checkout
+# for another time for the symlink thing.
+
+git config --global core.symlinks true
+git config --global core.autocrlf false
+git clone /c/workdir ray
+cd ray
+
+git checkout -f "${BUILDKITE_COMMIT}"
+
+export PYTHON="3.8"
+export RAY_USE_RANDOM_PORTS="1"
+export RAY_DEFAULT_BUILD="1"
+export LC_ALL="en_US.UTF-8"
+export LANG="en_US.UTF-8"
+export BUILD="1"
+export DL="1"
+
+powershell ci/pipeline/fix-windows-container-networking.ps1
+powershell ci/pipeline/fix-windows-bazel.ps1
+
+readonly PIPELINE_POSTMERGE="0189e759-8c96-4302-b6b5-b4274406bf89"
+
+cleanup() {
+    if [[ "${BUILDKITE_PIPELINE_ID:-}" == "${PIPELINE_POSTMERGE}" ]]; then
+        bash ./ci/build/upload_build_info.sh
+    fi
+}
+trap cleanup EXIT
+
+upload_wheels() {
+    if [[ "${BUILDKITE_PIPELINE_ID:-}" != "${PIPELINE_POSTMERGE}" ]]; then
+        exit 0
+    fi
+
+    pip install -q docker aws_requests_auth boto3
+    python .buildkite/copy_files.py --destination branch_wheels --path python/dist
+    if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+        python .buildkite/copy_files.py --destination wheels --path python/dist
+    fi
+}
+
+if [[ "$1" == "wheels" ]]; then
+    export WINDOWS_WHEELS=1
+    bash ci/ci.sh init
+    bash ci/ci.sh build
+    upload_wheels
+    exit 0
+fi
+
+# Run tests
+
+conda init
+bash ci/ci.sh init
+bash ci/ci.sh build
+export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1
+
+bash ci/ci.sh "$1"

ci/ci.sh

@@ -243,7 +243,6 @@ test_python() {
   if [ "${OSTYPE}" = msys ]; then
     args+=(
       python/ray/tests/...
-      python/ray/train:test_windows
       -python/ray/tests:test_actor_advanced  # crashes in shutdown
       -python/ray/tests:test_autoscaler # We don't support Autoscaler on Windows
       -python/ray/tests:test_autoscaler_aws
@@ -290,6 +289,33 @@ test_python() {
   fi
 }
 
+# For running Python tests on Windows (excluding Serve).
+test_train_windows() {
+  if [ "${OSTYPE}" = msys ]; then
+    args+=(
+      python/ray/train:test_windows
+    )
+  fi
+  if [ 0 -lt "${#args[@]}" ]; then  # Any targets to test?
+    install_ray
+
+    # Shard the args.
+    BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-'0'}
+    BUILDKITE_PARALLEL_JOB_COUNT=${BUILDKITE_PARALLEL_JOB_COUNT:-'1'}
+    test_shard_selection=$(python ./ci/ray_ci/bazel_sharding.py --exclude_manual --index "${BUILDKITE_PARALLEL_JOB}" --count "${BUILDKITE_PARALLEL_JOB_COUNT}" "${args[@]}")
+
+    # shellcheck disable=SC2046,SC2086
+    bazel test --config=ci \
+      --build_tests_only $(./ci/run/bazel_export_options) \
+      --test_env=CI="1" \
+      --test_env=RAY_CI_POST_WHEEL_TESTS="1" \
+      --test_env=USERPROFILE="${USERPROFILE}" \
+      --test_output=streamed \
+      -- \
+      ${test_shard_selection};
+  fi
+}
+
 # For running large Python tests on Linux and MacOS.
 test_large() {
   # shellcheck disable=SC2046

python/build-wheel-windows.sh

@@ -5,11 +5,7 @@ set -euxo pipefail
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
 WORKSPACE_DIR="${ROOT_DIR}/.."
 
-PY_VERSIONS=("3.8"
-             "3.9"
-             "3.10"
-             "3.11"
-             )
+PY_VERSIONS=("3.8" "3.9" "3.10" "3.11")
 
 bazel_preclean() {
   "${WORKSPACE_DIR}"/ci/run/bazel.py preclean "mnemonic(\"Genrule\", deps(//:*))"
@@ -94,7 +90,10 @@ build_wheel_windows() {
 
   local local_dir="python/dist"
   for pyversion in "${PY_VERSIONS[@]}"; do
-    if [ -z "${pyversion}" ]; then continue; fi
+    if [[ "${BUILD_ONE_PYTHON_ONLY:-}" != "" && "${pyversion}" != "${BUILD_ONE_PYTHON_ONLY}" ]]; then
+      continue
+    fi
+
     bazel_preclean
     git clean -q -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client
     git checkout -q -f -- .