ARROW-1253: [C++/Python] Speed up C++ / Python builds by using conda-…

…forge toolchain for thirdparty libraries

@pcmoritz I removed the travis_script_plasma.sh and combined the common bits with travis_script_python.sh; it seemed like there was some redundant work and would be easier to test everything in a single build. Let me know if I messed something up

Author: Wes McKinney <[email protected]>

Closes apache#884 from wesm/ARROW-1253 and squashes the following commits:

30a13ee [Wes McKinney] Formatting
28bc6be [Wes McKinney] Run unit tests from installed library
fcf7940 [Wes McKinney] Prevent existing LD_LIBRARY_PATH from being blown away
c67bb68 [Wes McKinney] Speed up C++ / Python builds by using conda-forge toolchain to avoid rebuilding EPs. Use ninja for builds when using toolchain

.travis.yml

@@ -42,7 +42,6 @@ cache:
   ccache: true
   directories:
     - $HOME/.conda_packages
-    - $HOME/.ccache
 
 matrix:
   fast_finish: true
@@ -56,6 +55,9 @@ matrix:
     before_script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
+    - export ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - export ARROW_TRAVIS_VALGRIND=1
+    - export ARROW_TRAVIS_PLASMA=1
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -66,6 +68,8 @@ matrix:
     cache:
     addons:
     before_script:
+    - export ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - export ARROW_TRAVIS_PLASMA=1
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -120,27 +124,6 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
-  - compiler: gcc
-    language: cpp
-    os: linux
-    group: deprecated
-    before_script:
-    - export CC="gcc-4.9"
-    - export CXX="g++-4.9"
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
-  - compiler: clang
-    osx_image: xcode6.4
-    os: osx
-    cache:
-    addons:
-    before_script:
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
 
 before_install:
 - ulimit -c unlimited -S

ci/travis_before_script_cpp.sh

@@ -15,22 +15,33 @@
 
 set -ex
 
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
 if [ "$1" == "--only-library" ]; then
   only_library_mode=yes
 else
   only_library_mode=no
+  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 fi
 
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-
-if [ $only_library_mode == "no" ]; then
-  # C++ toolchain
-  export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-  export RAPIDJSON_HOME=$CPP_TOOLCHAIN
-
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
   # Set up C++ toolchain from conda-forge packages for faster builds
-  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 rapidjson
+  conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
+        jemalloc=4.4.0 \
+        nomkl \
+        boost-cpp \
+        rapidjson \
+        flatbuffers \
+        gflags \
+        lz4-c \
+        snappy \
+        zstd \
+        brotli \
+        zlib \
+        cmake \
+        curl \
+        thrift-cpp \
+        ninja
 fi
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
@@ -45,7 +56,6 @@ pushd $ARROW_CPP_BUILD_DIR
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
--DARROW_PLASMA=ON \
 -DARROW_NO_DEPRECATED_API=ON"
 CMAKE_LINUX_FLAGS=""
 CMAKE_OSX_FLAGS=""
@@ -60,8 +70,20 @@ else
   # also in the manylinux1 image.
   CMAKE_LINUX_FLAGS="\
 $CMAKE_LINUX_FLAGS \
--DARROW_JEMALLOC=ON \
--DARROW_TEST_MEMCHECK=ON"
+-DARROW_JEMALLOC=ON"
+fi
+
+# Use Ninja for faster builds when using toolchain
+if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja"
+fi
+
+if [ $ARROW_TRAVIS_PLASMA == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON"
+fi
+
+if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON"
 fi
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
@@ -76,7 +98,7 @@ else
           $ARROW_CPP_DIR
 fi
 
-make VERBOSE=1 -j4
-make install
+$TRAVIS_MAKE -j4
+$TRAVIS_MAKE install
 
 popd

ci/travis_env_common.sh

@@ -29,6 +29,19 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
 export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
 export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
 
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
+  # C++ toolchain
+  export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+  export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+  export BOOST_ROOT=$CPP_TOOLCHAIN
+
+  export PATH=$CPP_TOOLCHAIN/bin:$PATH
+  export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+  export TRAVIS_MAKE=ninja
+else
+  export TRAVIS_MAKE=make
+fi
+
 if [ $TRAVIS_OS_NAME == "osx" ]; then
   export GOPATH=$TRAVIS_BUILD_DIR/gopath
 fi

ci/travis_script_cpp.sh

@@ -14,15 +14,15 @@
 
 set -e
 
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 # Check licenses according to Apache policy
 git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
 ./dev/release/run-rat.sh arrow-src.tar.gz
 
 pushd $CPP_BUILD_DIR
 
-make lint
+$TRAVIS_MAKE lint
 
 # ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
 # until we are able to install the full LLVM toolchain in Travis CI again