Update arrow using Plasma with glog (ray-project#2913)

* Update Arrow to Plasma with glog and update the building process

* Remove ParquetExternalProject.cmake

* Fix Mac building error in CI

* Use find_package(BISON) instead of hard code

* Revert BISON binary to hard code.

* Remove build_parquet.sh

* Update setup.sh

cmake/Modules/ArrowExternalProject.cmake

@@ -14,7 +14,10 @@
 #  - PLASMA_SHARED_LIB
 
 set(arrow_URL https://github.com/apache/arrow.git)
-set(arrow_TAG 927bd34aaad875e82beca2584d5d777839fa8bb0)
+# The PR for this commit is https://github.com/apache/arrow/pull/2522. We
+# include the link here to make it easier to find the right commit because
+# Arrow often rewrites git history and invalidates certain commits.
+set(arrow_TAG 7104d64ff2cd6c20e29d3cf4ec5c58bc10798f66)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})
@@ -53,11 +56,27 @@ set(ARROW_CMAKE_ARGS
   -DARROW_JEMALLOC=off
   -DARROW_WITH_BROTLI=off
   -DARROW_WITH_LZ4=off
-  -DARROW_WITH_ZLIB=off
   -DARROW_WITH_ZSTD=off
   -DFLATBUFFERS_HOME=${FLATBUFFERS_HOME}
   -DBOOST_ROOT=${BOOST_ROOT})
 
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  # PyArrow needs following settings.
+  set(ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS}
+    -DARROW_WITH_THRIFT=ON
+    -DARROW_PARQUET=ON
+    -DARROW_WITH_ZLIB=ON)
+else()
+  set(ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS}
+    -DARROW_WITH_THRIFT=OFF
+    -DARROW_PARQUET=OFF
+    -DARROW_WITH_ZLIB=OFF)
+endif ()
+if (APPLE)
+  set(ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS}
+    -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison)
+endif()
+
 if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
   set(ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} -DARROW_PLASMA_JAVA_CLIENT=ON)
 endif ()

cmake/Modules/ThirdpartyToolchain.cmake

@@ -150,19 +150,6 @@ ADD_THIRDPARTY_LIB(plasma STATIC_LIB ${PLASMA_STATIC_LIB})
 add_dependencies(plasma plasma_ep)
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-  # Apache parquet cpp
-  include(ParquetExternalProject)
-
-  message(STATUS "Parquet home: ${PARQUET_HOME}")
-  message(STATUS "Parquet include dir: ${PARQUET_INCLUDE_DIR}")
-  message(STATUS "Parquet static library: ${PARQUET_STATIC_LIB}")
-  message(STATUS "Parquet shared library: ${PARQUET_SHARED_LIB}")
-  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
-
-  ADD_THIRDPARTY_LIB(parquet STATIC_LIB ${PARQUET_STATIC_LIB})
-
-  add_dependencies(parquet parquet_ep)
-
   # pyarrow
   find_package(PythonInterp REQUIRED)
   message(STATUS "PYTHON_EXECUTABLE for pyarrow: ${PYTHON_EXECUTABLE}")
@@ -185,7 +172,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # add_custom_command would have problem with setup.py
   ExternalProject_Add(pyarrow_ext
     PREFIX external/pyarrow
-    DEPENDS parquet_ep
+    DEPENDS arrow_ep
     DOWNLOAD_COMMAND ""
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build

thirdparty/scripts/setup.sh

@@ -57,11 +57,6 @@ bash "$TP_SCRIPT_DIR/build_credis.sh"
 #RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
 #bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
 
-##############################################
-# parquet (skipped as it is inlined in build_arrow.sh)
-##############################################
-# bash "$TP_SCRIPT_DIR/build_parquet.sh"
-
 ##############################################
 # catapult
 ##############################################