ARROW-4611: [C++] Rework CMake logic

This changes refactors much of our CMake logic to make use of built-in CMake paths and remove custom logic. It also switches to the use of more modern dependency management via CMake targets instead of plain text variables. This includes the following fixes: - Use CMake's standard find features, e.g. respecting the `*_ROOT` variables: https://issues.apache.org/jira/browse/ARROW-4383 - Add a Dockerfile for Fedora: https://issues.apache.org/jira/browse/ARROW-4730 - Add a Dockerfile for Ubuntu Xenial: https://issues.apache.org/jira/browse/ARROW-4731 - Add a Dockerfile for Ubuntu Bionic: https://issues.apache.org/jira/browse/ARROW-4849 - Add a Dockerfile for Debian Testing: https://issues.apache.org/jira/browse/ARROW-4732 - Change the clang-7 entry to use system packages without any dependency on conda(-forge): https://issues.apache.org/jira/browse/ARROW-4733 - Support `double-conversion<3.1`: https://issues.apache.org/jira/browse/ARROW-4617 - Use google benchmark from toolchain: https://issues.apache.org/jira/browse/ARROW-4609 - Use the `compilers` metapackage to install the correct binutils when using conda, otherwise system binutils to fix https://issues.apache.org/jira/browse/ARROW-4485 - RapidJSON throws compiler errors with GCC 8+ https://issues.apache.org/jira/browse/ARROW-4750 - Handle `EXPECT_OK` collision: https://issues.apache.org/jira/browse/ARROW-4760 - Activate flight build in ci/docker_build_cpp.sh: https://issues.apache.org/jira/browse/ARROW-4614 - Build Gandiva in the docker containers: https://issues.apache.org/jira/browse/ARROW-4644 Author: Uwe L. Korn <[email protected]> Closes apache#3688 from xhochy/build-on-fedora and squashes the following commits: 88e11fc <Uwe L. Korn> ARROW-4611: Rework CMake logic
davidhao3300 · Mar 14, 2019 · 5114be6 · 5114be6
1 parent 2b574f9
commit 5114be6
Show file tree

Hide file tree

Showing 106 changed files with 3,459 additions and 3,176 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -56,16 +56,15 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-  - name: "C++ unit tests w/ Valgrind, clang 7.0"
+  - name: "C++ unit tests w/ clang 7.0, system packages"
     os: linux
     env:
-    - ARROW_TRAVIS_VALGRIND=1
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
-    - ARROW_TRAVIS_FLIGHT=1
+    - ARROW_TRAVIS_USE_SYSTEM=1
     - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_ORC=1
     - ARROW_TRAVIS_PARQUET=1
     - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_VERBOSE=1
     - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - CC="clang-7"
@@ -80,56 +79,25 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
   # Separating Valgrind and C++ coverage makes individual jobs shorter
-  - name: "C++ unit tests w/ gcc 5.4, coverage"
+  - name: "C++ unit tests w/ conda-forge toolchain, coverage"
     compiler: gcc
     language: cpp
     os: linux
     jdk: openjdk8
     env:
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - ARROW_TRAVIS_COVERAGE=1
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_TRAVIS_FLIGHT=1
-    - ARROW_TRAVIS_PLASMA=1
-    - ARROW_TRAVIS_ORC=1
-    - ARROW_TRAVIS_PARQUET=1
     - ARROW_TRAVIS_GANDIVA=1
     - ARROW_TRAVIS_GANDIVA_JAVA=1
-    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
-    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    before_script:
-    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
-    # If either C++ or Python changed, we must install the C++ libraries
-    - git submodule update --init
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
-    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
-    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
-  - name: "C++ unit tests w/ gcc 4.8, trusty"
-    dist: trusty
-    compiler: gcc
-    language: cpp
-    os: linux
-    jdk: openjdk8
-    env:
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
-    - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_ORC=1
     - ARROW_TRAVIS_PARQUET=1
-    - ARROW_TRAVIS_GANDIVA=1
-    - ARROW_TRAVIS_GANDIVA_JAVA=1
-    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    before_install:
-    - ulimit -c unlimited -S
-    - |
-        if [ $TRAVIS_OS_NAME == "linux" ]; then
-          sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
-          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-          sudo apt-get update -qq
-        fi
-    - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
+    - ARROW_TRAVIS_PLASMA=1
+    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    # TODO: This fails in re2 code
+    # - ARROW_TRAVIS_VALGRIND=1
+    - ARROW_TRAVIS_VERBOSE=1
     before_script:
     - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
@@ -140,21 +108,23 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
-  - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage"
+    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
+  - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage"
     compiler: gcc
     language: cpp
     os: linux
     jdk: openjdk8
     env:
     # Valgrind is needed for the Plasma store tests
-    - ARROW_TRAVIS_VALGRIND=1
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - ARROW_TRAVIS_COVERAGE=1
+    - ARROW_TRAVIS_FLIGHT=1
+    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
     - ARROW_TRAVIS_PYTHON_DOCS=1
     - ARROW_TRAVIS_PYTHON_JVM=1
-    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
-    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_VALGRIND=1
     # TODO(wesm): Run the benchmarks outside of Travis
     # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
     before_script:
@@ -172,10 +142,9 @@ matrix:
     - export PLASMA_VALGRIND=1
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
-  - name: "[OS X] C++ w/ XCode 8.3"
+  - name: "[OS X] C++ w/ XCode 9.3"
     compiler: clang
-    language: cpp
-    osx_image: xcode8.3
+    osx_image: xcode9.3
     os: osx
     cache:
     addons:
@@ -185,41 +154,38 @@ matrix:
     - ARROW_TRAVIS_FLIGHT=1
     - ARROW_TRAVIS_ORC=1
     - ARROW_TRAVIS_PARQUET=1
-    - ARROW_TRAVIS_GANDIVA=1
-    - ARROW_TRAVIS_GANDIVA_JAVA=1
+    # TODO(ARROW-4763): llvm and llvmdev packages are in conflict:
+    # https://github.com/conda-forge/llvmdev-feedstock/issues/60
+    # - ARROW_TRAVIS_GANDIVA=1
+    # - ARROW_TRAVIS_GANDIVA_JAVA=1
     - ARROW_TRAVIS_OPTIONAL_INSTALL=1
-    - ARROW_TRAVIS_VERBOSE=0
+    - ARROW_TRAVIS_VERBOSE=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib
-    # which can get loaded before the toolchain Boost libraries. These seem to
-    # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own
-    # Boost and statically link to get around the issue until this can be
-    # investigated further
-    - ARROW_TRAVIS_VENDORED_BOOST=1
     before_script:
     - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     # If either C++ or Python changed, we must install the C++ libraries
     - git submodule update --init
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
-    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
-  - name: "[OS X] Python w/ XCode 7.3"
+    # Disabled because of ARROW-4763
+    # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
+  - name: "[OS X] Python w/ XCode 9.3"
     compiler: clang
-    language: cpp
-    osx_image: xcode7.3
+    osx_image: xcode9.3
     os: osx
     cache:
     addons:
     env:
+    - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    - ARROW_TRAVIS_VERBOSE=1
     - MACOSX_DEPLOYMENT_TARGET="10.9"
     before_script:
     script:
     - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   - name: "[manylinux1] Python"
@@ -256,16 +222,15 @@ matrix:
     - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
-  - name: "Integration w/ OpenJDK 8, clang 7"
+  - name: "Integration w/ OpenJDK 8, conda-forge toolchain"
     language: java
     os: linux
     env: ARROW_TEST_GROUP=integration
     jdk: openjdk8
     env:
     - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1
-    - CC="clang-7"
-    - CXX="clang++-7"
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
     before_script:
     - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
@@ -275,7 +240,8 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh
+    # TODO(ARROW-4764): Using system java and plasma built with conda doesn't work
+    # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh
   - name: "NodeJS"
     language: node_js
     os: linux
@@ -289,7 +255,6 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
   - name: "C++ & GLib & Ruby w/ gcc 5.4"
     compiler: gcc
-    language: cpp
     os: linux
     env:
     - ARROW_TRAVIS_ORC=1
@@ -369,6 +334,7 @@ matrix:
     dist: trusty
     env:
     - ARROW_TRAVIS_PARQUET=1
+    - ARROW_TRAVIS_USE_SYSTEM=1
     before_install:
     # Have to copy-paste this here because of how R's build steps work
     - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
@@ -379,6 +345,7 @@ matrix:
           sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
           sudo apt-get update -qq
         fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
     - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib

diff --git a/LICENSE.txt b/LICENSE.txt
@@ -837,3 +837,13 @@ This project includes code from the manylinux project.
 Copyright: 2016 manylinux
 Homepage: https://github.com/pypa/manylinux
 License: The MIT License (MIT)
+
+--------------------------------------------------------------------------------
+
+This project include code from CMake.
+
+* cpp/cmake_modules/FindGTest.cmake is based on code from CMake.
+
+Copyright: Copyright 2000-2019 Kitware, Inc. and Contributors
+Homepage: https://gitlab.kitware.com/cmake/cmake
+License: 3-clause BSD
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -49,6 +49,10 @@ This product includes software from Google Guava (Apache 2.0)
   * Copyright (C) 2007 The Guava Authors
   * https://github.com/google/guava
 
+This product include software from CMake (BSD 3-Clause)
+  * CMake - Cross Platform Makefile Generator
+  * Copyright 2000-2019 Kitware, Inc. and Contributors
+
 The web site includes files generated by Jekyll.
 
 --------------------------------------------------------------------------------

diff --git a/appveyor.yml b/appveyor.yml
@@ -34,8 +34,8 @@ cache:
   - C:\Users\Appveyor\clcache
   - C:\Users\Appveyor\.cargo\registry
 
-matrix:
-  fast_finish: true
+#matrix:
+#  fast_finish: true
 
 environment:
   global:
@@ -61,10 +61,6 @@ environment:
     - JOB: "Build_Debug"
       GENERATOR: Ninja
       CONFIGURATION: "Debug"
-    - JOB: "Cmake_Script_Tests"
-      GENERATOR: Ninja
-      CONFIGURATION: "Release"
-      BUILD_SCRIPT: "CMake_Build_Script"
     - JOB: "MinGW32"
       MINGW_PACKAGE_PREFIX: mingw-w64-i686
       MINGW_PREFIX: c:\msys64\mingw32

diff --git a/ci/appveyor-build.bat b/ci/appveyor-build.bat
@@ -28,8 +28,6 @@ git config core.symlinks true
 git reset --hard
 if "%JOB:~,5%" == "MinGW" (
     call ci\appveyor-cpp-build-mingw.bat
-) else if "%JOB%" == "Cmake_Script_Tests" (
-    call ci\appveyor-cpp-test-cmake-script.bat
 ) else (
     call ci\appveyor-cpp-build.bat
 )

diff --git a/ci/appveyor-cpp-build-mingw.bat b/ci/appveyor-cpp-build-mingw.bat
@@ -32,17 +32,12 @@ set CPP_BUILD_DIR=cpp\build
 mkdir %CPP_BUILD_DIR%
 pushd %CPP_BUILD_DIR%
 
-set BOOST_ROOT=%MINGW_PREFIX%
-set LZ4_HOME=%MINGW_PREFIX%
-set ZSTD_HOME=%MINGW_PREFIX%
-set SNAPPY_HOME=%MINGW_PREFIX%
-set BROTLI_HOME=%MINGW_PREFIX%
-set FLATBUFFERS_HOME=%MINGW_PREFIX%
 cmake ^
     -G "MSYS Makefiles" ^
     -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% ^
     -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
     -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
+    -DARROW_PACKAGE_PREFIX=%MINGW_PREFIX% ^
     -DARROW_JEMALLOC=OFF ^
     -DARROW_USE_GLOG=OFF ^
     -DARROW_PYTHON=ON ^

diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
@@ -30,7 +30,7 @@ if "%JOB%" == "Static_Crt_Build" (
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
-        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
+        -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^
         -DARROW_USE_STATIC_CRT=ON ^
         -DARROW_BOOST_USE_SHARED=OFF ^
         -DARROW_BUILD_SHARED=OFF ^
@@ -108,7 +108,7 @@ if "%JOB%" == "Toolchain" (
   set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml
 )
 
-conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge
+conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge || exit /B
 
 call activate arrow
 

diff --git a/ci/appveyor-cpp-setup-mingw.bat b/ci/appveyor-cpp-setup-mingw.bat
@@ -23,6 +23,7 @@ pacman -S --noconfirm ^
     "%MINGW_PACKAGE_PREFIX%-boost" ^
     "%MINGW_PACKAGE_PREFIX%-brotli" ^
     "%MINGW_PACKAGE_PREFIX%-cmake" ^
+    "%MINGW_PACKAGE_PREFIX%-double-conversion" ^
     "%MINGW_PACKAGE_PREFIX%-flatbuffers" ^
     "%MINGW_PACKAGE_PREFIX%-gcc" ^
     "%MINGW_PACKAGE_PREFIX%-gflags" ^
@@ -32,6 +33,7 @@ pacman -S --noconfirm ^
     "%MINGW_PACKAGE_PREFIX%-meson" ^
     "%MINGW_PACKAGE_PREFIX%-protobuf" ^
     "%MINGW_PACKAGE_PREFIX%-python3-numpy" ^
+    "%MINGW_PACKAGE_PREFIX%-rapidjson" ^
     "%MINGW_PACKAGE_PREFIX%-snappy" ^
     "%MINGW_PACKAGE_PREFIX%-zlib" ^
     "%MINGW_PACKAGE_PREFIX%-zstd" || exit /B