[serve] Implement timeouts for streaming & enable all tests with expe…

…rimental streaming flag turned on (ray-project#36261) This is the penultimate PR for turning RAY_SERVE_EXPERIMENTAL_STREAMING on by default. In this change, we are now running all Serve tests under three conditions: - RAY_SERVE_EXPERIMENTAL_STREAMING=0 && RAY_SERVE_ENABLE_NEW_ROUTING=0 - RAY_SERVE_EXPERIMENTAL_STREAMING=0 && RAY_SERVE_ENABLE_NEW_ROUTING=1 - RAY_SERVE_EXPERIMENTAL_STREAMING=1 && RAY_SERVE_ENABLE_NEW_ROUTING=1 (Note that RAY_SERVE_EXPERIMENTAL_STREAMING=1 implies RAY_SERVE_ENABLE_NEW_ROUTING so the latter is not explicitly set). This required making two functionality fixes: - Never using the streaming path for Java even if the flag is on. - Implementing timeouts in the streaming path. I could split these into separate PRs, but they can't really be fully tested independently, so opting to combine them here.
Mark2000 · Jun 30, 2023 · f40d236 · f40d236
1 parent fa385e7
commit f40d236
Show file tree

Hide file tree

Showing 24 changed files with 600 additions and 282 deletions.
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -10,11 +10,11 @@
   commands:
     - ./java/test.sh
 
-- label: ":java: Java (RAY_SERVE_ENABLE_NEW_ROUTING=1)"
+- label: ":java: Java (RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1)"
   conditions: ["RAY_CI_JAVA_AFFECTED"]
   instance_size: medium
   commands:
-    - export RAY_SERVE_ENABLE_NEW_ROUTING=1 && ./java/test.sh
+    - export RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1 && ./java/test.sh
 
 - label: ":serverless: Dashboard Tests"
   conditions:
@@ -110,6 +110,39 @@
       --test_env=RAY_SERVE_ENABLE_NEW_ROUTING=1
       $(cat test_shard.txt)
 
+- label: ":serverless: Serve Tests (RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1)"
+  parallelism: 3
+  conditions:
+    [
+        "RAY_CI_SERVE_AFFECTED",
+        "RAY_CI_PYTHON_AFFECTED",
+        "RAY_CI_ML_AFFECTED",
+    ]
+  instance_size: large
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - TORCH_VERSION=1.9.0 ./ci/env/install-dependencies.sh
+    - bash ./ci/ci.sh prepare_docker
+    - 'git clone https://github.com/wg/wrk.git /tmp/wrk && pushd /tmp/wrk && make -j && sudo cp wrk /usr/local/bin && popd'
+    - ./ci/env/env_info.sh
+    - >-
+      set -x;
+      python ./ci/run/bazel_sharding/bazel_sharding.py
+      --exclude_manual
+      --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}"
+      --tag_filters=-post_wheel_build,-gpu
+      python/ray/serve/...
+      > test_shard.txt
+    - cat test_shard.txt
+    - bazel test --config=ci $(./ci/run/bazel_export_options)
+      --test_tag_filters=-post_wheel_build,-gpu
+      --test_env=DOCKER_HOST=tcp://docker:2376
+      --test_env=DOCKER_TLS_VERIFY=1
+      --test_env=DOCKER_CERT_PATH=/certs/client
+      --test_env=DOCKER_TLS_CERTDIR=/certs
+      --test_env=RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1
+      $(cat test_shard.txt)
+
 - label: ":python: Minimal install Python {{matrix}}"
   conditions: ["RAY_CI_PYTHON_AFFECTED"]
   instance_size: medium

diff --git a/.buildkite/pipeline.build_py37.yml b/.buildkite/pipeline.build_py37.yml
@@ -142,7 +142,7 @@
       $(cat test_shard.txt)
 
 
-- label: ":cold_face: :python: :serverless: Python 3.7 Serve Tests (RAY_SERVE_USE_NEW_ROUTING=1)"
+- label: ":cold_face: :python: :serverless: Python 3.7 Serve Tests (RAY_SERVE_ENABLE_NEW_ROUTING=1)"
   parallelism: 3
   conditions:
     [
@@ -173,5 +173,40 @@
       --test_env=DOCKER_TLS_VERIFY=1
       --test_env=DOCKER_CERT_PATH=/certs/client
       --test_env=DOCKER_TLS_CERTDIR=/certs
-      --test_env=RAY_SERVE_USE_NEW_ROUTING=1
+      --test_env=RAY_SERVE_ENABLE_NEW_ROUTING=1
+      $(cat test_shard.txt)
+
+
+- label: ":cold_face: :python: :serverless: Python 3.7 Serve Tests (RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1)"
+  parallelism: 3
+  conditions:
+    [
+        "RAY_CI_SERVE_AFFECTED",
+        "RAY_CI_PYTHON_AFFECTED",
+        "RAY_CI_ML_AFFECTED",
+    ]
+  instance_size: large
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-minimal.sh 3.7
+    - PYTHON=3.7 TORCH_VERSION=1.9.0 ./ci/env/install-dependencies.sh
+    - bash ./ci/ci.sh prepare_docker
+    - 'git clone https://github.com/wg/wrk.git /tmp/wrk && pushd /tmp/wrk && make -j && sudo cp wrk /usr/local/bin && popd'
+    - ./ci/env/env_info.sh
+    - >-
+      set -x;
+      python ./ci/run/bazel_sharding/bazel_sharding.py
+      --exclude_manual
+      --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}"
+      --tag_filters=-post_wheel_build,-gpu
+      python/ray/serve/...
+      > test_shard.txt
+    - cat test_shard.txt
+    - bazel test --config=ci $(./ci/run/bazel_export_options)
+      --test_tag_filters=-post_wheel_build,-gpu
+      --test_env=DOCKER_HOST=tcp://docker:2376
+      --test_env=DOCKER_TLS_VERIFY=1
+      --test_env=DOCKER_CERT_PATH=/certs/client
+      --test_env=DOCKER_TLS_CERTDIR=/certs
+      --test_env=RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1
       $(cat test_shard.txt)
diff --git a/python/ray/serve/BUILD b/python/ray/serve/BUILD
@@ -312,6 +312,14 @@ py_test(
     deps = [":serve_lib"],
 )
 
+py_test(
+    name = "test_request_timeout",
+    size = "medium",
+    srcs = serve_tests_srcs,
+    tags = ["exclusive", "team:serve"],
+    deps = [":serve_lib"],
+)
+
 py_test(
     name = "test_standalone",
     size = "large",
@@ -494,29 +502,12 @@ py_test(
    deps = [":serve_lib"],
 )
 
-# Runs a subset of the tests with experimental streaming turned on.
-py_test(
-   name = "test_experimental_streaming",
-   size = "large",
-   srcs = glob(["tests/test_experimental_streaming.py",
-               "tests/test_api.py",
-               "tests/test_failure.py",
-               "tests/test_fastapi.py",
-               "tests/test_http_adapters.py",
-               "tests/test_http_headers.py",
-               "**/conftest.py"]),
-   tags = ["exclusive", "team:serve"],
-   deps = [":serve_lib"],
-   env = {"RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING": "1"},
-)
-
 py_test(
     name = "test_streaming_response",
     size = "large",
     srcs = serve_tests_srcs,
     tags = ["exclusive", "team:serve"],
     deps = [":serve_lib"],
-    env = {"RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING": "1"},
 )
 
 py_test(
@@ -525,7 +516,6 @@ py_test(
     srcs = serve_tests_srcs,
     tags = ["exclusive", "team:serve"],
     deps = [":serve_lib"],
-    env = {"RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING": "1"},
 )
 
 py_test(
@@ -613,7 +603,7 @@ py_test(
 
 py_test(
     name = "test_gradio",
-    size = "small",
+    size = "medium",
     srcs = serve_tests_srcs,
     tags = ["exclusive", "team:serve"],
     deps = [":serve_lib"],

diff --git a/python/ray/serve/_private/application_state.py b/python/ray/serve/_private/application_state.py
@@ -193,9 +193,14 @@ def apply_deployment_info(
         self._deployment_state_manager.deploy(deployment_name, deployment_info)
 
         if deployment_info.route_prefix is not None:
+            config = deployment_info.deployment_config
             self._endpoint_state.update_endpoint(
                 deployment_name,
-                EndpointInfo(route=deployment_info.route_prefix, app_name=self._name),
+                EndpointInfo(
+                    route=deployment_info.route_prefix,
+                    app_name=self._name,
+                    app_is_cross_language=config.is_cross_language,
+                ),
             )
         else:
             self._endpoint_state.delete_endpoint(deployment_name)

diff --git a/python/ray/serve/_private/client.py b/python/ray/serve/_private/client.py
@@ -308,7 +308,14 @@ def deploy(
             route_prefix=route_prefix,
         )
 
-        updating = ray.get(self._controller.deploy.remote(**controller_deploy_args))
+        updating = ray.get(
+            self._controller.deploy.remote(
+                # TODO(edoakes): this is a hack because the deployment_language
+                # doesn't seem to get set properly from Java.
+                is_deployed_from_python=True,
+                **controller_deploy_args,
+            )
+        )
 
         tag = self.log_deployment_update_status(name, version, updating)
 

diff --git a/python/ray/serve/_private/common.py b/python/ray/serve/_private/common.py
@@ -28,6 +28,7 @@
 class EndpointInfo:
     route: str
     app_name: str
+    app_is_cross_language: bool = False
 
 
 # Keep in sync with ServeReplicaState in dashboard/client/src/type/serve.ts