[Streaming]Farewell : remove all of streaming related from ray repo. (r…

…ay-project#21770) New repo url is https://github.com/ray-project/mobius Co-authored-by: 林濯 <[email protected]>
Mark2000 · Jan 23, 2022 · ec62d7f · ec62d7f
1 parent 2da2ac5
commit ec62d7f
Show file tree

Hide file tree

Showing 412 changed files with 13,509 additions and 36,499 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -3,7 +3,6 @@
     [
         "RAY_CI_LINUX_WHEELS_AFFECTED",
         "RAY_CI_JAVA_AFFECTED",
-        "RAY_CI_STREAMING_JAVA_AFFECTED",
     ]
   commands:
     # Build the wheels and jars
@@ -41,7 +40,6 @@
     [
         "RAY_CI_LINUX_WHEELS_AFFECTED",
         "RAY_CI_JAVA_AFFECTED",
-        "RAY_CI_STREAMING_JAVA_AFFECTED",
     ]
   commands:
     # Build the debug wheels
@@ -61,7 +59,6 @@
 #     [
 #         "RAY_CI_LINUX_WHEELS_AFFECTED",
 #         "RAY_CI_JAVA_AFFECTED",
-#         "RAY_CI_STREAMING_JAVA_AFFECTED",
 #     ]
 #   commands:
 #     # Build the asan wheels
@@ -156,24 +153,6 @@
   commands:
     - RAY_bootstrap_with_gcs=1 RAY_gcs_grpc_based_pubsub=1 RAY_gcs_storage=memory ./java/test.sh
 
-- label: ":java: Streaming"
-  conditions:
-    ["RAY_CI_STREAMING_PYTHON_AFFECTED", "RAY_CI_STREAMING_JAVA_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
-  commands:
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      //streaming:all
-    - bash streaming/src/test/run_streaming_queue_test.sh
-
-- label: ":java: :redis: Streaming"
-  conditions:
-    ["RAY_CI_STREAMING_PYTHON_AFFECTED", "RAY_CI_STREAMING_JAVA_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
-  commands:
-    - bazel test --config=ci
-      --test_env=RAY_gcs_grpc_based_pubsub=1
-      --test_env=RAY_bootstrap_with_gcs=1
-      --test_env=RAY_gcs_storage=memory $(./scripts/bazel_export_options) //streaming:all
-    - RAY_bootstrap_with_gcs=1 RAY_gcs_grpc_based_pubsub=1 RAY_gcs_storage=memory bash streaming/src/test/run_streaming_queue_test.sh
-
 - label: ":cpp: Ray CPP Worker"
   conditions: [ "RAY_CI_CPP_AFFECTED" ]
   commands:

diff --git a/.flake8 b/.flake8
@@ -1,7 +1,6 @@
 [flake8]
 exclude =
   python/ray/core/generated/
-  streaming/python/generated
   doc/source/conf.py
   python/ray/cloudpickle/
   python/ray/thirdparty_files/

diff --git a/.gitignore b/.gitignore
@@ -169,20 +169,10 @@ dependency-reduced-pom.xml
 # Cpp
 cpp/example/thirdparty/
 
-# streaming/python
-streaming/python/generated/
-streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/generated/
-streaming/build/java
 .clwb
-streaming/**/.settings
-streaming/java/**/target
-streaming/java/**/.classpath
-streaming/java/**/.project
-streaming/java/**/*.log
 
 # pom.xml files generated from pom_template.xml
 java/**/pom.xml
-streaming/java/**/pom.xml
 
 # python virtual env
 venv

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -450,18 +450,18 @@ cc_library(
     ],
 )
 
-# This header is used to warp some streaming code so we can reduce suspicious
+# This header is used to warp some internal code so we can reduce suspicious
 # symbols export.
 cc_library(
-    name = "exported_streaming_internal",
+    name = "exported_internal",
     srcs = glob(
         [
-            "src/ray/streaming/streaming.cc",
+            "src/ray/internal/internal.cc",
         ],
     ),
     hdrs = glob(
         [
-            "src/ray/streaming/streaming.h",
+            "src/ray/internal/internal.h",
         ],
     ),
     copts = COPTS,
@@ -674,7 +674,7 @@ cc_library(
         ],
     }),
     strip_include_prefix = "src",
-    visibility = ["//streaming:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [
         ":agent_manager_rpc",
         ":gcs",
@@ -722,7 +722,7 @@ cc_library(
         ],
     }),
     strip_include_prefix = "src",
-    visibility = ["//streaming:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [
         ":agent_manager_rpc",
         ":node_manager_fbs",
@@ -2337,7 +2337,7 @@ pyx_library(
     ),
     deps = [
         "//:core_worker_lib",
-        "//:exported_streaming_internal",
+        "//:exported_internal",
         "//:global_state_accessor_lib",
         "//:ray_util",
         "//:raylet_lib",
@@ -2348,33 +2348,6 @@ pyx_library(
     ],
 )
 
-pyx_library(
-    name = "_streaming",
-    srcs = glob([
-        "python/ray/streaming/_streaming.pyx",
-        "python/ray/__init__.py",
-        "python/ray/_raylet.pxd",
-        "python/ray/includes/*.pxd",
-        "python/ray/includes/*.pxi",
-        "python/ray/streaming/__init__.pxd",
-        "python/ray/streaming/includes/*.pxd",
-        "python/ray/streaming/includes/*.pxi",
-    ]),
-    cc_kwargs = dict(
-        srcs = PYX_SRCS,
-        copts = COPTS + PYX_COPTS,
-    ),
-    deps = [
-        "//streaming:streaming_lib",
-        "@com_github_grpc_grpc//:grpc++",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
-        "@com_google_absl//absl/memory",
-        "@com_google_absl//absl/synchronization",
-        "@com_google_absl//absl/time",
-    ],
-)
-
 cc_binary(
     name = "libcore_worker_library_java.so",
     srcs = glob([
@@ -2398,7 +2371,7 @@ cc_binary(
     visibility = ["//java:__subpackages__"],
     deps = [
         "//:core_worker_lib",
-        "//:exported_streaming_internal",
+        "//:exported_internal",
         "//:global_state_accessor_lib",
         "//:src/ray/ray_exported_symbols.lds",
         "//:src/ray/ray_version_script.lds",
@@ -2491,12 +2464,6 @@ copy_to_workspace(
     dstdir = "python/ray",
 )
 
-copy_to_workspace(
-    name = "cp_streaming",
-    srcs = ["python/ray/streaming/_streaming.so"],
-    dstdir = "python/ray/streaming",
-)
-
 copy_to_workspace(
     name = "cp_all_py_proto",
     srcs = [":all_py_proto"],
@@ -2535,7 +2502,6 @@ genrule(
     srcs = [
         ":cp_all_py_proto",
         ":cp_serve_py_proto",
-        "//streaming:copy_streaming_py_proto",
     ],
     outs = ["install_py_proto.out"],
     cmd = """
@@ -2555,7 +2521,6 @@ genrule(
     name = "ray_pkg",
     srcs = [
         ":cp_raylet_so",
-        ":cp_streaming",
         ":python_sources",
         ":install_py_proto",
         ":cp_redis",

diff --git a/bazel/ray_deps_build_all.bzl b/bazel/ray_deps_build_all.bzl
@@ -1,6 +1,5 @@
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps")
-load("@com_github_ray_project_ray//streaming/java:dependencies.bzl", "gen_streaming_java_deps")
 load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
 load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories")
 load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure")
@@ -12,7 +11,6 @@ load("@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl", "jar_jar_repositories")
 def ray_deps_build_all():
   bazel_skylib_workspace()
   gen_java_deps()
-  gen_streaming_java_deps()
   boost_deps()
   prometheus_cpp_repositories()
   python_configure(name = "local_config_python")

diff --git a/ci/travis/bazel-format.sh b/ci/travis/bazel-format.sh
@@ -45,6 +45,6 @@ done
 
 pushd "$ROOT_DIR"/../..
 BAZEL_FILES=(bazel/BUILD bazel/ray.bzl BUILD.bazel java/BUILD.bazel \
- cpp/BUILD.bazel cpp/example/BUILD.bazel streaming/BUILD.bazel streaming/java/BUILD.bazel WORKSPACE)
+ cpp/BUILD.bazel cpp/example/BUILD.bazel WORKSPACE)
 buildifier -mode=$RUN_TYPE -diff_command="diff -u" "${BAZEL_FILES[@]}"
 popd
diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py
@@ -88,9 +88,6 @@ def get_commit_range():
     RAY_CI_PYTHON_AFFECTED = 0
     RAY_CI_LINUX_WHEELS_AFFECTED = 0
     RAY_CI_MACOS_WHEELS_AFFECTED = 0
-    RAY_CI_STREAMING_CPP_AFFECTED = 0
-    RAY_CI_STREAMING_PYTHON_AFFECTED = 0
-    RAY_CI_STREAMING_JAVA_AFFECTED = 0
     RAY_CI_DASHBOARD_AFFECTED = 0
     RAY_CI_DOCKER_AFFECTED = 0
     RAY_CI_DOC_AFFECTED = 0
@@ -179,7 +176,6 @@ def get_commit_range():
                 RAY_CI_DASHBOARD_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
-                RAY_CI_STREAMING_PYTHON_AFFECTED = 1
                 RAY_CI_DOC_AFFECTED = 1
                 # Python changes might impact cross language stack in Java.
                 # Java also depends on Python CLI to manage processes.
@@ -189,7 +185,6 @@ def get_commit_range():
                     RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 1
             elif changed_file.startswith("java/"):
                 RAY_CI_JAVA_AFFECTED = 1
-                RAY_CI_STREAMING_JAVA_AFFECTED = 1
             elif changed_file.startswith("cpp/"):
                 RAY_CI_CPP_AFFECTED = 1
             elif changed_file.startswith("docker/"):
@@ -218,19 +213,8 @@ def get_commit_range():
                 RAY_CI_PYTHON_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
-                RAY_CI_STREAMING_CPP_AFFECTED = 1
-                RAY_CI_STREAMING_PYTHON_AFFECTED = 1
-                RAY_CI_STREAMING_JAVA_AFFECTED = 1
                 RAY_CI_DASHBOARD_AFFECTED = 1
                 RAY_CI_DOC_AFFECTED = 1
-            elif changed_file.startswith("streaming/src"):
-                RAY_CI_STREAMING_CPP_AFFECTED = 1
-                RAY_CI_STREAMING_PYTHON_AFFECTED = 1
-                RAY_CI_STREAMING_JAVA_AFFECTED = 1
-            elif changed_file.startswith("streaming/python"):
-                RAY_CI_STREAMING_PYTHON_AFFECTED = 1
-            elif changed_file.startswith("streaming/java"):
-                RAY_CI_STREAMING_JAVA_AFFECTED = 1
             else:
                 RAY_CI_TUNE_AFFECTED = 1
                 RAY_CI_SGD_AFFECTED = 1
@@ -244,9 +228,6 @@ def get_commit_range():
                 RAY_CI_DOC_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
-                RAY_CI_STREAMING_CPP_AFFECTED = 1
-                RAY_CI_STREAMING_PYTHON_AFFECTED = 1
-                RAY_CI_STREAMING_JAVA_AFFECTED = 1
                 RAY_CI_DASHBOARD_AFFECTED = 1
     else:
         RAY_CI_TUNE_AFFECTED = 1
@@ -262,9 +243,6 @@ def get_commit_range():
         RAY_CI_DOC_AFFECTED = 1
         RAY_CI_LINUX_WHEELS_AFFECTED = 1
         RAY_CI_MACOS_WHEELS_AFFECTED = 1
-        RAY_CI_STREAMING_CPP_AFFECTED = 1
-        RAY_CI_STREAMING_PYTHON_AFFECTED = 1
-        RAY_CI_STREAMING_JAVA_AFFECTED = 1
         RAY_CI_DASHBOARD_AFFECTED = 1
 
     # Log the modified environment variables visible in console.
@@ -284,12 +262,6 @@ def get_commit_range():
         "RAY_CI_PYTHON_AFFECTED={}".format(RAY_CI_PYTHON_AFFECTED),
         "RAY_CI_LINUX_WHEELS_AFFECTED={}".format(RAY_CI_LINUX_WHEELS_AFFECTED),
         "RAY_CI_MACOS_WHEELS_AFFECTED={}".format(RAY_CI_MACOS_WHEELS_AFFECTED),
-        "RAY_CI_STREAMING_CPP_AFFECTED={}".format(
-            RAY_CI_STREAMING_CPP_AFFECTED),
-        "RAY_CI_STREAMING_PYTHON_AFFECTED={}".format(
-            RAY_CI_STREAMING_PYTHON_AFFECTED),
-        "RAY_CI_STREAMING_JAVA_AFFECTED={}".format(
-            RAY_CI_STREAMING_JAVA_AFFECTED),
         "RAY_CI_DOCKER_AFFECTED={}".format(RAY_CI_DOCKER_AFFECTED),
         "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED={}".format(
             RAY_CI_PYTHON_DEPENDENCIES_AFFECTED),

diff --git a/java/BUILD.bazel b/java/BUILD.bazel
@@ -312,8 +312,6 @@ java_binary(
         "//java:io_ray_ray_api",
         "//java:io_ray_ray_runtime",
         "//java:io_ray_ray_serve",
-        "//streaming/java:io_ray_ray_streaming-api",
-        "//streaming/java:io_ray_ray_streaming-runtime",
     ],
 )
 
@@ -335,7 +333,6 @@ genrule(
     srcs = [
         "//java:ray_dist_shaded.jar",
         "//java:gen_maven_deps",
-        "//streaming/java:gen_maven_deps",
     ],
     outs = ["ray_java_pkg.out"],
     cmd = """

diff --git a/python/ray/streaming b/python/ray/streaming
diff --git a/python/ray/streaming/__init__.py b/python/ray/streaming/__init__.py
@@ -0,0 +1,26 @@
+# flake8: noqa
+# Ray should be imported before streaming
+import ray
+
+
+# Redirect ray.streaming.* to raystreaming, so user can keep
+# their original habit.
+def _update_modules():
+    try:
+        import raystreaming
+        import raystreaming.context
+        import sys
+        ray_streaming_module_name = raystreaming.__name__
+        ray_streaming_modules = {}
+        for mod_name, module in sys.modules.items():
+            if mod_name.startswith(ray_streaming_module_name):
+                ray_streaming_modules[mod_name.replace(
+                    "raystreaming", "ray.streaming")] = module
+        sys.modules.update(ray_streaming_modules)
+    except Exception as e:
+        print("import raystreaming error: ", e)
+
+
+_update_modules()
+
+__all__ = ["StreamingContext"]
diff --git a/src/ray/streaming/streaming.cc → src/ray/internal/internal.cc b/src/ray/streaming/streaming.cc → src/ray/internal/internal.cc
@@ -12,12 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "ray/streaming/streaming.h"
+#include "ray/internal/internal.h"
 
 #include "ray/core_worker/core_worker.h"
 
 namespace ray {
-namespace streaming {
+namespace internal {
 
 using ray::core::CoreWorkerProcess;
 using ray::core::TaskOptions;
@@ -52,5 +52,5 @@ std::vector<rpc::ObjectReference> SendInternal(const ActorID &peer_actor_id,
   }
   return result.value();
 }
-}  // namespace streaming
+}  // namespace internal
 }  // namespace ray
diff --git a/src/ray/streaming/streaming.h → src/ray/internal/internal.h b/src/ray/streaming/streaming.h → src/ray/internal/internal.h
@@ -17,10 +17,10 @@
 #include "ray/common/id.h"
 #include "ray/core_worker/common.h"
 
-// This header is used to warp some streaming code so we can reduce suspicious
+// This header is used to warp some internal code so we can reduce suspicious
 // symbols export.
 namespace ray {
-namespace streaming {
+namespace internal {
 
 using ray::core::RayFunction;
 
@@ -33,5 +33,5 @@ std::vector<rpc::ObjectReference> SendInternal(const ActorID &peer_actor_id,
                                                std::shared_ptr<LocalMemoryBuffer> buffer,
                                                RayFunction &function, int return_num);
 
-}  // namespace streaming
+}  // namespace internal
 }  // namespace ray