add 100 job submissions test (ray-project#39079)

* add 100 job submissions test Signed-off-by: Ruiyang Wang <[email protected]> * make trivial change to pythons Signed-off-by: Ruiyang Wang <[email protected]> --------- Signed-off-by: Ruiyang Wang <[email protected]>
cread · Aug 30, 2023 · 313c146 · 313c146
1 parent 4788e4f
commit 313c146
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 1 deletion.
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -334,6 +334,58 @@
     - echo "--- Running the script with fault of networking delay"
     - ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=1000 --sleep-secs=0.01
 
+
+- label: ":kubernetes: :mending_heart: :runner: chaos network delay many job submissions"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: medium
+  commands:
+    - |
+      cleanup() {
+        if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi
+        kind delete cluster
+      }
+      trap cleanup EXIT
+    - ./ci/env/install-minimal.sh 3.8
+    - PYTHON=3.8 ./ci/env/install-dependencies.sh
+    # Specifying above somehow messes up the Ray install.
+    # Uninstall and re-install Ray so that we can use Ray Client.
+    # (Remove thirdparty_files to sidestep an issue with psutil.)
+    - pip uninstall -y ray && rm -rf /ray/python/ray/thirdparty_files
+    - pip install -e /ray/python
+    - echo "--- Setting up local kind cluster."
+    - ./ci/k8s/prep-k8s-environment.sh
+    - ./ci/k8s/prep-helm.sh
+    - echo "--- Building py38-cpu Ray image for the test."
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker
+    - python ci/build/build-docker-images.py --py-versions py38 --device-types cpu --build-type LOCAL --build-base
+    # Tag the image built in the last step. We want to be sure to distinguish the image from the real Ray nightly.
+    - docker tag rayproject/ray:nightly-py38-cpu ray-ci:kuberay-test
+    # Load the image into the kind node
+    - kind load docker-image ray-ci:kuberay-test
+    # Helm install KubeRay
+    - echo "--- Installing KubeRay operator and cluser."
+    - helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+    - helm install kuberay-operator kuberay/kuberay-operator
+    - kubectl wait pod  -l app.kubernetes.io/name=kuberay-operator --for=condition=Ready=True  --timeout=5m
+    # We are in m4i.xlarge and have 4 cpus. Can't have too many nodes.
+    - helm install raycluster kuberay/ray-cluster --set image.repository=ray-ci --set image.tag=kuberay-test --set worker.replicas=2 --set worker.resources.limits.cpu=500m --set worker.resources.requests.cpu=500m --set head.resources.limits.cpu=500m --set head.resources.requests.cpu=500m
+    - kubectl wait pod -l ray.io/cluster=raycluster-kuberay --for=condition=Ready=True --timeout=5m
+    - kubectl port-forward --address 0.0.0.0 service/raycluster-kuberay-head-svc 8265:8265 &
+    # Helm install chaos-mesh
+    - echo "--- Installing chaos-mesh operator and CR."
+    - helm repo add chaos-mesh https://charts.chaos-mesh.org
+    - kubectl create ns chaos-mesh
+    - helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-mesh --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --version 2.6.1
+    - kubectl wait pod  --namespace chaos-mesh  -l app.kubernetes.io/instance=chaos-mesh --for=condition=Ready=True
+    - echo "--- Running the script without faults"
+    - ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=3 --sleep-secs=0.01
+    # Now add the delay, rerun the job
+    - kubectl apply -f python/ray/tests/chaos/chaos_network_delay.yaml
+    - echo "--- Running the script 100 times with fault of networking delay"
+    - for i in {1..100}; do ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=3 --sleep-secs=0.01; done
+
+
 # TODO: write a test that needs bandwidth heavy lifting
 - label: ":kubernetes: :mending_heart: chaos bandwidth test"
   conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]

diff --git a/python/ray/tests/chaos/streaming_llm.py b/python/ray/tests/chaos/streaming_llm.py
@@ -41,7 +41,7 @@ def __init__(self, llm):
 
     @fastapi_app.post("/")
     async def handle_request(self, prompt: str) -> StreamingResponse:
-        logger.info(f'Got prompt: "{prompt}"')
+        logger.info(f'Got prompt with size "{len(prompt)}"')
         remote_async_gen = await self.llm.remote(prompt)
         return StreamingResponse(
             self.local_async_gen(remote_async_gen), media_type="text/plain"