[JavaWorker] Changes to the build system for support java worker (ray…

…-project#2092)

* Changes to the build system for support java worker
--------------------------
This commit includes changes to the build system, which is part of the java worker support of Ray.
It consists of the following changes:
 - the changes of CMakeLists.txt files
 - the changes of the python setup.py and init files for the adaptation of the changed build system
 - move the location of local_scheduler_extension.cc for the adaptation of the changed build system which maybe better support multi-language worker

* minor whitespace

* Linting

CMakeLists.txt

@@ -2,6 +2,19 @@ cmake_minimum_required(VERSION 3.4)
 
 project(ray)
 
+set(CMAKE_RAY_LANG_PYTHON "NO")
+set(CMAKE_RAY_LANG_JAVA "NO")
+if ("${CMAKE_RAY_LANGUAGE}" STREQUAL "python")
+  set(CMAKE_RAY_LANG_PYTHON "YES")
+elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "java")
+  set(CMAKE_RAY_LANG_JAVA "YES")
+elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "")
+  message(WARNING "Language is not set, choose Python as default.")
+  set(CMAKE_RAY_LANG_PYTHON "YES")
+else()
+  message(FATAL_ERROR "Unrecognized language, use -DCMAKE_RAY_LANGUAGE=java|python. Abort.")
+endif()
+
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake)

build.sh

@@ -7,14 +7,19 @@ set -e
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
-if [[ -z  "$1" ]]; then
-  PYTHON_EXECUTABLE=`which python`
-else
-  PYTHON_EXECUTABLE=$1
-fi
-echo "Using Python executable $PYTHON_EXECUTABLE."
-
-bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE
+function usage()
+{
+  echo "Usage: build.sh [<args>]"
+  echo
+  echo "Options:"
+  echo "  -h|--help               print the help info"
+  echo "  -d|--debug              CMAKE_BUILD_TYPE=Debug (default is RelWithDebInfo)"
+  echo "  -l|--language python(default) "
+  echo "                          build native library for python"
+  echo "                java      build native library for java"
+  echo "  -p|--python             which python executable (default from which python)"
+  echo
+}
 
 # Determine how many parallel jobs to use for make based on the number of cores
 unamestr="$(uname)"
@@ -27,30 +32,83 @@ else
   exit 1
 fi
 
+LANGUAGE="python"
+PYTHON_EXECUTABLE=""
+BUILD_DIR=""
+if [ "$VALGRIND" = "1" ]; then
+  CBUILD_TYPE="Debug"
+else
+  CBUILD_TYPE="RelWithDebInfo"
+fi
+
+# Parse options
+while [[ $# > 0 ]]; do
+  key="$1"
+  case $key in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -d|--debug)
+      CBUILD_TYPE=Debug
+      ;;
+    -l|--languags)
+      LANGUAGE="$2"
+      if [ "$LANGUAGE" != "python" ] && [ "$LANGUAGE" != "java" ]; then
+        echo "Unrecognized language."
+        exit -1
+      fi
+      shift
+      ;;
+    -p|--python)
+      PYTHON_EXECUTABLE="$2"
+      shift
+      ;;
+    *)
+      echo "ERROR: unknown option \"$key\""
+      echo
+      usage
+      exit -1
+      ;;
+  esac
+  shift
+done
+
+if [[ -z  "$PYTHON_EXECUTABLE" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
+bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE
+
 # Now we build everything.
-pushd "$ROOT_DIR/python/ray/core"
-  # We use these variables to set PKG_CONFIG_PATH, which is important so that
-  # in cmake, pkg-config can find plasma.
-  TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
-  ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
-  if [[ "$VALGRIND" = "1" ]]; then
-    BOOST_ROOT=$TP_PKG_DIR/boost \
-    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
-    cmake -DCMAKE_BUILD_TYPE=Debug \
-          -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-          -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
-          ../../..
-  else
-    BOOST_ROOT=$TP_PKG_DIR/boost \
-    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
-    cmake -DCMAKE_BUILD_TYPE=Release \
-          -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-          -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
-          ../../..
+if [[ "$LANGUAGE" == "java" ]]; then
+  BUILD_DIR="$ROOT_DIR/build/"
+  if [ ! -d "${BUILD_DIR}" ]; then
+  mkdir -p ${BUILD_DIR}
   fi
-  make clean
-  make -j${PARALLEL}
+else
+  BUILD_DIR="$ROOT_DIR/python/ray/core"
+fi
+
+pushd "$BUILD_DIR"
+TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
+# We use these variables to set PKG_CONFIG_PATH, which is important so that
+# in cmake, pkg-config can find plasma.
+ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
+BOOST_ROOT=$TP_PKG_DIR/boost \
+PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
+cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
+      -DCMAKE_RAY_LANGUAGE=$LANGUAGE \
+      -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
+      -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE $ROOT_DIR
+
+make clean
+make -j${PARALLEL}
 popd
 
 # Move stuff from Arrow to Ray.
-cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $ROOT_DIR/python/ray/core/src/plasma/
+cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/
+if [[ "$LANGUAGE" == "java" ]]; then
+  cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/
+fi

python/ray/local_scheduler/__init__.py

@@ -2,7 +2,7 @@
 from __future__ import division
 from __future__ import print_function
 
-from ray.core.src.local_scheduler.liblocal_scheduler_library import (
+from ray.core.src.local_scheduler.liblocal_scheduler_library_python import (
     Task, LocalSchedulerClient, ObjectID, check_simple_value, task_from_string,
     task_to_string, _config, common_error)
 from .local_scheduler_services import start_local_scheduler

python/setup.py

@@ -20,7 +20,7 @@
     "ray/core/src/common/redis_module/libray_redis_module.so",
     "ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager",
     "ray/core/src/local_scheduler/local_scheduler",
-    "ray/core/src/local_scheduler/liblocal_scheduler_library.so",
+    "ray/core/src/local_scheduler/liblocal_scheduler_library_python.so",
     "ray/core/src/global_scheduler/global_scheduler",
     "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
     "ray/WebUI.ipynb"
@@ -63,7 +63,7 @@ def run(self):
         # version of Python to build pyarrow inside the build.sh script. Note
         # that certain flags will not be passed along such as --user or sudo.
         # TODO(rkn): Fix this.
-        subprocess.check_call(["../build.sh", sys.executable])
+        subprocess.check_call(["../build.sh", "-p", sys.executable])
 
         # We also need to install pyarrow along with Ray, so make sure that the
         # relevant non-Python pyarrow files get copied.

src/common/CMakeLists.txt

@@ -29,22 +29,43 @@ add_custom_command(
   COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
   VERBATIM)
 
-add_custom_target(gen_common_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
-
-# Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
-add_custom_command(
-  TARGET gen_common_fbs
-  COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
-  DEPENDS ${FBS_DEPENDS}
-  COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
-  VERBATIM)
-
-# Encode the fact that the ray redis module requires the autogenerated
-# flatbuffer files to compile.
-add_dependencies(ray_redis_module gen_common_fbs)
-
-add_dependencies(gen_common_fbs flatbuffers_ep)
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
+
+  # Generate Python bindings for the flatbuffers objects.
+  set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+  add_custom_command(
+    TARGET gen_common_python_fbs
+    COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
+    DEPENDS ${FBS_DEPENDS}
+    COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
+    VERBATIM)
+
+  # Encode the fact that the ray redis module requires the autogenerated
+  # flatbuffer files to compile.
+  add_dependencies(ray_redis_module gen_common_python_fbs)
+
+  add_dependencies(gen_common_python_fbs flatbuffers_ep)
+endif()
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
+  add_custom_target(gen_common_java_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
+
+  # Generate Java bindings for the flatbuffers objects.
+  set(JAVA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/java)
+  add_custom_command(
+    TARGET gen_common_java_fbs
+    COMMAND ${FLATBUFFERS_COMPILER} -j -o ${JAVA_OUTPUT_DIR} ${COMMON_FBS_SRC}
+    DEPENDS ${FBS_DEPENDS}
+    COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
+    VERBATIM)
+
+  # Encode the fact that the ray redis module requires the autogenerated
+  # flatbuffer files to compile.
+  add_dependencies(ray_redis_module gen_common_java_fbs)
+
+  add_dependencies(gen_common_java_fbs flatbuffers_ep)
+endif()
 
 add_custom_target(
    hiredis
@@ -71,7 +92,13 @@ add_library(common STATIC
   thirdparty/ae/ae.c
   thirdparty/sha256.c)
 
-add_dependencies(common gen_common_fbs)
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  add_dependencies(common gen_common_python_fbs)
+endif()
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
+  add_dependencies(common gen_common_java_fbs)
+endif()
 
 target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a")
 

src/common/cmake/Common.cmake

@@ -43,17 +43,33 @@ include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
 
 set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -fPIC -std=c99")
 
-# Code for finding Python
-find_package(PythonInterp REQUIRED)
-find_package(NumPy REQUIRED)
-
-# Now find the Python include directories.
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
-                OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
-message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
-
-message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
-message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
+# language-specific
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  # Code for finding Python
+  find_package(PythonInterp REQUIRED)
+  find_package(NumPy REQUIRED)
+
+  # Now find the Python include directories.
+  execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
+                  OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
+  message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
+
+  message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
+  message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
+endif ()
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
+  find_package(JNI REQUIRED)
+  # add jni support
+  include_directories(${JAVA_INCLUDE_PATH})
+  include_directories(${JAVA_INCLUDE_PATH2})
+  if (JNI_FOUND)
+    message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+    message (STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
+  else()
+    message (WARNING "NOT FIND JNI")
+  endif()
+endif()
 
 # Common libraries
 
@@ -62,4 +78,6 @@ set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a"
 
 include_directories("${CMAKE_CURRENT_LIST_DIR}/..")
 include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/")
-include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
+endif ()