diff --git a/.travis.yml b/.travis.yml index 9cc2b86c05cde..b93f1c2519b9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,6 @@ cache: ccache: true directories: - $HOME/.conda_packages - - $HOME/.ccache matrix: fast_finish: true @@ -56,6 +55,9 @@ matrix: before_script: - export CC="gcc-4.9" - export CXX="g++-4.9" + - export ARROW_TRAVIS_USE_TOOLCHAIN=1 + - export ARROW_TRAVIS_VALGRIND=1 + - export ARROW_TRAVIS_PLASMA=1 - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh @@ -66,6 +68,8 @@ matrix: cache: addons: before_script: + - export ARROW_TRAVIS_USE_TOOLCHAIN=1 + - export ARROW_TRAVIS_PLASMA=1 - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh @@ -120,27 +124,6 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh - - compiler: gcc - language: cpp - os: linux - group: deprecated - before_script: - - export CC="gcc-4.9" - - export CXX="g++-4.9" - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh - - compiler: clang - osx_image: xcode6.4 - os: osx - cache: - addons: - before_script: - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh before_install: - ulimit -c unlimited -S diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index e250e705f1fbd..a1697dae59fbf 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -15,22 +15,33 @@ set -ex +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh + if [ "$1" == "--only-library" ]; then only_library_mode=yes else only_library_mode=no + source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh fi -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -if [ $only_library_mode == "no" ]; then - # C++ toolchain - export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain - export RAPIDJSON_HOME=$CPP_TOOLCHAIN - +if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then # Set up C++ toolchain from conda-forge packages for faster builds - source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh - conda create -y -q -p $CPP_TOOLCHAIN python=2.7 rapidjson + conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \ + jemalloc=4.4.0 \ + nomkl \ + boost-cpp \ + rapidjson \ + flatbuffers \ + gflags \ + lz4-c \ + snappy \ + zstd \ + brotli \ + zlib \ + cmake \ + curl \ + thrift-cpp \ + ninja fi if [ $TRAVIS_OS_NAME == "osx" ]; then @@ -45,7 +56,6 @@ pushd $ARROW_CPP_BUILD_DIR CMAKE_COMMON_FLAGS="\ -DARROW_BUILD_BENCHMARKS=ON \ -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \ --DARROW_PLASMA=ON \ -DARROW_NO_DEPRECATED_API=ON" CMAKE_LINUX_FLAGS="" CMAKE_OSX_FLAGS="" @@ -60,8 +70,20 @@ else # also in the manylinux1 image. CMAKE_LINUX_FLAGS="\ $CMAKE_LINUX_FLAGS \ --DARROW_JEMALLOC=ON \ --DARROW_TEST_MEMCHECK=ON" +-DARROW_JEMALLOC=ON" +fi + +# Use Ninja for faster builds when using toolchain +if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja" +fi + +if [ $ARROW_TRAVIS_PLASMA == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON" +fi + +if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON" fi if [ $TRAVIS_OS_NAME == "linux" ]; then @@ -76,7 +98,7 @@ else $ARROW_CPP_DIR fi -make VERBOSE=1 -j4 -make install +$TRAVIS_MAKE -j4 +$TRAVIS_MAKE install popd diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh index a2e591014cfd3..4122eacaa2ee3 100755 --- a/ci/travis_env_common.sh +++ b/ci/travis_env_common.sh @@ -29,6 +29,19 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install +if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then + # C++ toolchain + export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain + export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN + export BOOST_ROOT=$CPP_TOOLCHAIN + + export PATH=$CPP_TOOLCHAIN/bin:$PATH + export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH + export TRAVIS_MAKE=ninja +else + export TRAVIS_MAKE=make +fi + if [ $TRAVIS_OS_NAME == "osx" ]; then export GOPATH=$TRAVIS_BUILD_DIR/gopath fi diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh index c368a1daeddb9..a4f6b056d9de4 100755 --- a/ci/travis_script_cpp.sh +++ b/ci/travis_script_cpp.sh @@ -14,7 +14,7 @@ set -e -: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build} +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh # Check licenses according to Apache policy git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz @@ -22,7 +22,7 @@ git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz pushd $CPP_BUILD_DIR -make lint +$TRAVIS_MAKE lint # ARROW-209: checks depending on the LLVM toolchain are disabled temporarily # until we are able to install the full LLVM toolchain in Travis CI again diff --git a/ci/travis_script_plasma.sh b/ci/travis_script_plasma.sh deleted file mode 100755 index fa384ade89c2f..0000000000000 --- a/ci/travis_script_plasma.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -set -e - -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - -export ARROW_HOME=$ARROW_CPP_INSTALL -export PYARROW_WITH_PLASMA=1 - -pushd $ARROW_PYTHON_DIR - -function build_arrow_libraries() { - CPP_BUILD_DIR=$1 - CPP_DIR=$TRAVIS_BUILD_DIR/cpp - - mkdir $CPP_BUILD_DIR - pushd $CPP_BUILD_DIR - - cmake -DARROW_BUILD_TESTS=off \ - -DARROW_PYTHON=on \ - -DARROW_PLASMA=on \ - -DCMAKE_INSTALL_PREFIX=$2 \ - $CPP_DIR - - make -j4 - make install - - popd -} - -python_version_tests() { - PYTHON_VERSION=$1 - CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION - - export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION - export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib - - conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl - source activate $CONDA_ENV_DIR - - python --version - which python - - # faster builds, please - conda install -y -q nomkl - - # Expensive dependencies install from Continuum package repo - conda install -y -q pip numpy pandas cython - - # Build C++ libraries - build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME - - # Other stuff pip install - pip install -r requirements.txt - - python setup.py build_ext --inplace - - python -m pytest -vv -r sxX pyarrow - - # Build documentation once - if [[ "$PYTHON_VERSION" == "3.6" ]] - then - conda install -y -q --file=doc/requirements.txt - python setup.py build_sphinx -s doc/source - fi - - # Build and install pyarrow - pushd $TRAVIS_BUILD_DIR/python - python setup.py install - popd - - # Run Plasma tests - pushd $TRAVIS_BUILD_DIR/python - python -m pytest pyarrow/tests/test_plasma.py - if [ $TRAVIS_OS_NAME == "linux" ]; then - PLASMA_VALGRIND=1 python -m pytest pyarrow/tests/test_plasma.py - fi - popd -} - -# run tests for python 2.7 and 3.6 -python_version_tests 2.7 -python_version_tests 3.6 - -popd diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index fdb5ad6a62c93..dcba78273caa7 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -17,26 +16,14 @@ set -e source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh export ARROW_HOME=$ARROW_CPP_INSTALL -export PYARROW_WITH_PLASMA=1 - -pushd $ARROW_PYTHON_DIR export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env +export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH build_parquet_cpp() { export PARQUET_ARROW_VERSION=$(git rev-parse HEAD) - conda create -y -q -p $PARQUET_HOME python=3.6 cmake curl - source activate $PARQUET_HOME - - # In case some package wants to download the MKL - conda install -y -q nomkl - - conda install -y -q thrift-cpp snappy zlib brotli boost - export BOOST_ROOT=$PARQUET_HOME - export SNAPPY_HOME=$PARQUET_HOME - export THRIFT_HOME=$PARQUET_HOME - export ZLIB_HOME=$PARQUET_HOME - export BROTLI_HOME=$PARQUET_HOME + # $CPP_TOOLCHAIN set up in before_script_cpp + export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet mkdir -p $PARQUET_DIR @@ -48,37 +35,39 @@ build_parquet_cpp() { cd build-dir cmake \ + -GNinja \ -DCMAKE_BUILD_TYPE=debug \ -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DPARQUET_BOOST_USE_SHARED=off \ -DPARQUET_BUILD_BENCHMARKS=off \ -DPARQUET_BUILD_EXECUTABLES=off \ - -DPARQUET_ZLIB_VENDORED=off \ - -DPARQUET_BUILD_TESTS=on \ + -DPARQUET_BUILD_TESTS=off \ .. - make -j${CPU_COUNT} - make install + ninja + ninja install popd } build_parquet_cpp -function build_arrow_libraries() { - CPP_BUILD_DIR=$1 - CPP_DIR=$TRAVIS_BUILD_DIR/cpp +function rebuild_arrow_libraries() { + pushd $ARROW_CPP_BUILD_DIR - mkdir $CPP_BUILD_DIR - pushd $CPP_BUILD_DIR + # Clear out prior build files + rm -rf * - cmake -DARROW_BUILD_TESTS=off \ + cmake -GNinja \ + -DARROW_BUILD_TESTS=off \ + -DARROW_BUILD_UTILITIES=off \ -DARROW_PLASMA=on \ -DARROW_PYTHON=on \ - -DCMAKE_INSTALL_PREFIX=$2 \ - $CPP_DIR + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + $ARROW_CPP_DIR - make -j4 - make install + ninja + ninja install popd } @@ -87,9 +76,6 @@ python_version_tests() { PYTHON_VERSION=$1 CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION - export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION - export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib - conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl source activate $CONDA_ENV_DIR @@ -103,27 +89,35 @@ python_version_tests() { conda install -y -q pip numpy pandas cython # Build C++ libraries - build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME + rebuild_arrow_libraries # Other stuff pip install + pushd $ARROW_PYTHON_DIR pip install -r requirements.txt - - python setup.py build_ext --inplace --with-parquet + python setup.py build_ext --with-parquet --with-plasma \ + install --single-version-externally-managed --record=record.text + popd python -c "import pyarrow.parquet" + python -c "import pyarrow.plasma" + + if [ $TRAVIS_OS_NAME == "linux" ]; then + export PLASMA_VALGRIND=1 + fi - python -m pytest -vv -r sxX pyarrow --parquet + PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow + python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet + pushd $ARROW_PYTHON_DIR # Build documentation once if [[ "$PYTHON_VERSION" == "3.6" ]] then conda install -y -q --file=doc/requirements.txt python setup.py build_sphinx -s doc/source fi + popd } # run tests for python 2.7 and 3.6 python_version_tests 2.7 python_version_tests 3.6 - -popd diff --git a/cpp/.gitignore b/cpp/.gitignore index 4910544ec87cd..ec846b35ba61a 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + thirdparty/ CMakeFiles/ CMakeCache.txt diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2891a5d76186c..41af1e16ce28d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -582,20 +582,6 @@ if (ARROW_STATIC_LINK_LIBS) add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS}) endif() -set(ARROW_MIN_TEST_LIBS - arrow_static - ${ARROW_STATIC_LINK_LIBS} - gtest - gtest_main) - -if(NOT MSVC) - set(ARROW_MIN_TEST_LIBS - ${ARROW_MIN_TEST_LIBS} - ${CMAKE_DL_LIBS}) -endif() - -set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS}) - set(ARROW_BENCHMARK_LINK_LIBS arrow_static arrow_benchmark_main @@ -618,6 +604,20 @@ if (NOT MSVC) ${CMAKE_DL_LIBS}) endif() +set(ARROW_MIN_TEST_LIBS + arrow_static + ${ARROW_STATIC_LINK_LIBS} + gtest + gtest_main) + +if(NOT MSVC) + set(ARROW_MIN_TEST_LIBS + ${ARROW_MIN_TEST_LIBS} + ${CMAKE_DL_LIBS}) +endif() + +set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS}) + if (ARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC) # In the case that jemalloc is only available as a shared library also use it to diff --git a/cpp/src/plasma/.gitignore b/cpp/src/plasma/.gitignore new file mode 100644 index 0000000000000..163b5c56e91a2 --- /dev/null +++ b/cpp/src/plasma/.gitignore @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +*_generated.h diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index 780aa4839610f..6554465466a55 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -225,7 +225,6 @@ In-Memory Object Store ObjectID PlasmaClient PlasmaBuffer - MutablePlasmaBuffer .. currentmodule:: pyarrow.parquet diff --git a/python/setup.py b/python/setup.py index 7425b71916001..be0e26b92a876 100644 --- a/python/setup.py +++ b/python/setup.py @@ -82,6 +82,7 @@ def run(self): user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'), ('build-type=', None, 'build type (debug or release)'), ('with-parquet', None, 'build the Parquet extension'), + ('with-plasma', None, 'build the Plasma extension'), ('bundle-arrow-cpp', None, 'bundle the Arrow C++ libraries')] + _build_ext.user_options) @@ -101,14 +102,13 @@ def initialize_options(self): os.environ.get('PYARROW_WITH_PARQUET', '0')) self.with_plasma = strtobool( os.environ.get('PYARROW_WITH_PLASMA', '0')) - if self.with_plasma and "plasma" not in self.CYTHON_MODULE_NAMES: - self.CYTHON_MODULE_NAMES.append("plasma") self.bundle_arrow_cpp = strtobool( os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0')) CYTHON_MODULE_NAMES = [ 'lib', - '_parquet'] + '_parquet', + 'plasma'] def _run_cmake(self): # The directory containing this setup.py