[build] Introduce an env variable to indicate the thirdparty path

Kudu thirdparty will cost dozens of minutes to build, now introduce an env variable THIRDPARTY_DIR to indicate the thirdparty path, it can be a well built thirdparty libraries path. With this feature, we can - Share the same thirdparty path by different code paths, these code paths are not needed to build thirdparty libraries repeatly - Build an docker image with thirdparty libraries well built, then we can use this image for building Kudu, CI, and etc. Change-Id: I5e0bdf4faa44322622c48cacadf8e1165eccd38a Reviewed-on: http://gerrit.cloudera.org:8080/18018 Tested-by: Kudu Jenkins Reviewed-by: Attila Bukor <[email protected]> Reviewed-by: Andrew Wong <[email protected]>
cutiechi · Nov 16, 2021 · dfb3ce9 · dfb3ce9
1 parent cb69bb1
commit dfb3ce9
Show file tree

Hide file tree

Showing 17 changed files with 66 additions and 37 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -90,8 +90,12 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 include(CMakeParseArguments)
 
 set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build-support)
-set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty)
-set(THIRDPARTY_TOOLCHAIN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/clang-toolchain)
+if ("$ENV{THIRDPARTY_DIR}" STREQUAL "")
+  set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty)
+else()
+  set(THIRDPARTY_DIR $ENV{THIRDPARTY_DIR})
+endif()
+set(THIRDPARTY_TOOLCHAIN_DIR ${THIRDPARTY_DIR}/clang-toolchain)
 set(THIRDPARTY_INSTALL_DIR ${THIRDPARTY_DIR}/installed)
 set(THIRDPARTY_INSTALL_COMMON_DIR ${THIRDPARTY_INSTALL_DIR}/common)
 set(THIRDPARTY_INSTALL_UNINSTRUMENTED_DIR ${THIRDPARTY_INSTALL_DIR}/uninstrumented)

diff --git a/build-support/ccache-clang/clang b/build-support/ccache-clang/clang
@@ -17,5 +17,6 @@
 # under the License.
 
 ROOT=$(dirname $BASH_SOURCE)/../..
-CLANG=$ROOT/thirdparty/clang-toolchain/bin/clang
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty}
+CLANG=${THIRDPARTY_DIR}/clang-toolchain/bin/clang
 CCACHE_CPP2=yes exec ccache $CLANG -Qunused-arguments `test -t 2 && echo -fcolor-diagnostics` "$@"
diff --git a/build-support/ccache-clang/clang++ b/build-support/ccache-clang/clang++
@@ -17,5 +17,6 @@
 # under the License.
 
 ROOT=$(dirname $BASH_SOURCE)/../..
-CLANG=$ROOT/thirdparty/clang-toolchain/bin/clang++
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty}
+CLANG=${THIRDPARTY_DIR}/clang-toolchain/bin/clang++
 CCACHE_CPP2=yes exec ccache $CLANG -Qunused-arguments `test -t 2 && echo -fcolor-diagnostics` "$@"
diff --git a/build-support/clang_tidy_gerrit.py b/build-support/clang_tidy_gerrit.py
@@ -33,16 +33,16 @@
 import unittest
 import tempfile
 
-from kudu_util import init_logging
+from kudu_util import init_logging, get_thirdparty_dir
 
 ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 
 BUILD_PATH = os.path.join(ROOT, "build", "latest")
 
 CLANG_TIDY_DIFF = os.path.join(
-    ROOT, "thirdparty/installed/uninstrumented/share/clang/clang-tidy-diff.py")
+    get_thirdparty_dir(), "installed/uninstrumented/share/clang/clang-tidy-diff.py")
 CLANG_TIDY = os.path.join(
-    ROOT, "thirdparty/clang-toolchain/bin/clang-tidy")
+    get_thirdparty_dir(), "clang-toolchain/bin/clang-tidy")
 
 GERRIT_USER = 'tidybot'
 GERRIT_PASSWORD = os.getenv('TIDYBOT_PASSWORD')

diff --git a/build-support/iwyu.py b/build-support/iwyu.py
@@ -30,7 +30,8 @@
 import subprocess
 import sys
 
-from kudu_util import get_upstream_commit, check_output, ROOT, Colors, init_logging
+from kudu_util import get_upstream_commit, check_output, ROOT, Colors, \
+  init_logging, get_thirdparty_dir
 import iwyu.fix_includes
 from iwyu.fix_includes import ParseAndMergeIWYUOutput
 
@@ -44,7 +45,7 @@
 """
 
 _MAPPINGS_DIR = os.path.join(ROOT, "build-support/iwyu/mappings/")
-_TOOLCHAIN_DIR = os.path.join(ROOT, "thirdparty/clang-toolchain/bin")
+_TOOLCHAIN_DIR = os.path.join(get_thirdparty_dir(), "clang-toolchain/bin")
 _IWYU_TOOL = os.path.join(ROOT, "build-support/iwyu/iwyu_tool.py")
 
 # Matches source files that we should run on.
@@ -129,7 +130,7 @@ def _relativize_paths(paths):
 
 
 def _get_thirdparty_include_dirs():
-  return glob.glob(os.path.join(ROOT, "thirdparty", "installed", "*", "include"))
+  return glob.glob(os.path.join(get_thirdparty_dir(), "installed", "*", "include"))
 
 
 def _get_fixer_flags(flags):

diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py
@@ -85,6 +85,8 @@
 import subprocess
 import sys
 import unittest
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from kudu_util import get_thirdparty_dir
 
 
 def iwyu_formatter(output):
@@ -232,7 +234,7 @@ def workaround_add_libcpp(build_dir, command):
     #
     # Some sanitizer builds (like TSAN) already include libc++; if the command
     # already has a flag including libc++, we don't want to add it again.
-    path_to_libcpp_prefix = os.path.join(build_dir, "..", "..", "thirdparty", "installed")
+    path_to_libcpp_prefix = os.path.join(get_thirdparty_dir(), "installed")
     path_to_libcpp_suffix = os.path.join("include", "c++", "v1")
     tp_pattern = os.path.join(path_to_libcpp_prefix, "*", path_to_libcpp_suffix)
     found = False

diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
@@ -149,12 +149,17 @@ if [ "$BUILD_TYPE" = "TSAN" ]; then
   export KUDU_USE_TSAN=1
 fi
 
+SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
+BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER
+
 export KUDU_FLAKY_TEST_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
 export KUDU_ALLOW_SLOW_TESTS=${KUDU_ALLOW_SLOW_TESTS:-$DEFAULT_ALLOW_SLOW_TESTS}
 export KUDU_COMPRESS_TEST_OUTPUT=${KUDU_COMPRESS_TEST_OUTPUT:-1}
 export TEST_TMPDIR=${TEST_TMPDIR:-/tmp/kudutest-$UID}
 export PARALLEL=${PARALLEL:-$(getconf _NPROCESSORS_ONLN)}
 export PARALLEL_TESTS=${PARALLEL_TESTS:-$PARALLEL}
+export THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty}
+
 BUILD_JAVA=${BUILD_JAVA:-1}
 BUILD_GRADLE=${BUILD_GRADLE:-1}
 BUILD_PYTHON=${BUILD_PYTHON:-1}
@@ -168,9 +173,6 @@ if [ ! -w "$TEST_TMPDIR" ]; then
   exit 1
 fi
 
-SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
-BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER
-
 # Remove testing artifacts from the previous run before we do anything
 # else. Otherwise, if we fail during the "build" step, Jenkins will
 # archive the test logs from the previous run, thinking they came from
@@ -312,16 +314,18 @@ if [ "$BUILD_TYPE" = "TSAN" ]; then
   THIRDPARTY_TYPE=tsan
 fi
 
-# The settings for PARALLEL (see above) also affects the thirdparty build.
-$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE
+if [ ! -n "${NO_REBUILD_THIRDPARTY}" ]; then
+  # The settings for PARALLEL (see above) also affects the thirdparty build.
+  $SOURCE_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_DIR/build-if-necessary.sh $THIRDPARTY_TYPE
+fi
 
-THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
+THIRDPARTY_BIN=$THIRDPARTY_DIR/installed/common/bin
 export PPROF_PATH=$THIRDPARTY_BIN/pprof
 
 if which ccache >/dev/null ; then
   CLANG=$(pwd)/build-support/ccache-clang/clang
 else
-  CLANG=$(pwd)/thirdparty/clang-toolchain/bin/clang
+  CLANG=$THIRDPARTY_DIR/clang-toolchain/bin/clang
 fi
 
 # Make sure we use JDK8
@@ -517,7 +521,7 @@ if [ "$DO_COVERAGE" == "1" ]; then
   echo
   echo Generating coverage report...
   echo ------------------------------------------------------------
-  if ! $SOURCE_ROOT/thirdparty/installed/common/bin/gcovr \
+  if ! $THIRDPARTY_DIR/installed/common/bin/gcovr \
       -r $SOURCE_ROOT \
       --gcov-filter='.*src#kudu.*' \
       --gcov-executable=$SOURCE_ROOT/build-support/llvm-gcov-wrapper \

diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
@@ -114,3 +114,8 @@ def get_my_email():
   """ Return the email address in the user's git config. """
   return check_output(['git', 'config', '--get',
       'user.email']).strip().decode('utf-8')
+
+
+def get_thirdparty_dir():
+  env = os.environ.copy()
+  return env['THIRDPARTY_DIR'] if env.has_key('THIRDPARTY_DIR') else os.path.join(ROOT, "thirdparty")
diff --git a/build-support/lint.sh b/build-support/lint.sh
@@ -50,7 +50,8 @@ fi
 cpplint_filter="+runtime/broken_libstdcpp_regex,-whitespace/comments,-readability/todo,-readability/inheritance,-build/header_guard,-build/include_order,-legal/copyright,-build/c++11,-readability/nolint"
 
 cd $ROOT
-$ROOT/thirdparty/installed/common/bin/cpplint.py \
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$ROOT/thirdparty}
+${THIRDPARTY_DIR}/installed/common/bin/cpplint.py \
   --verbose=4 \
   --filter=$cpplint_filter \
   $FILES 2>&1 | grep -v 'Done processing' | tee $TMP

diff --git a/build-support/llvm-gcov-wrapper b/build-support/llvm-gcov-wrapper
@@ -20,4 +20,5 @@
 # gcovr can call it.
 
 SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/..; pwd)
-exec $SOURCE_ROOT/thirdparty/clang-toolchain/bin/llvm-cov gcov "$@"
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty}
+exec $THIRDPARTY_DIR/clang-toolchain/bin/llvm-cov gcov "$@"
diff --git a/build-support/mini-cluster/build_mini_cluster_binaries.sh b/build-support/mini-cluster/build_mini_cluster_binaries.sh
@@ -84,13 +84,14 @@ if [ -d "$BUILD_ROOT" ]; then
   fi
 fi
 
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$BASE_DIR/thirdparty}
 cd $SOURCE_ROOT
 if [ -n "$NO_REBUILD_THIRDPARTY" ]; then
   echo Skipping thirdparty because NO_REBUILD_THIRDPARTY is not empty
 else
   echo Building thirdparty... >&2
   $SOURCE_ROOT/build-support/enable_devtoolset.sh \
-    ./thirdparty/build-if-necessary.sh
+    $THIRDPARTY_DIR/build-if-necessary.sh
 fi
 
 mkdir -p $BUILD_ROOT
@@ -121,7 +122,7 @@ rm -rf CMakeCache.txt CMakeFiles
 # with the mini-cluster.
 echo Configuring Kudu... >&2
 $SOURCE_ROOT/build-support/enable_devtoolset.sh \
-  $SOURCE_ROOT/thirdparty/installed/common/bin/cmake ../.. \
+  $THIRDPARTY_DIR/installed/common/bin/cmake ../.. \
   -DNO_TESTS=1 -DNO_CHRONY=1 \
   -DCMAKE_BUILD_TYPE=RELEASE -DKUDU_LINK=dynamic $EXTRA_CMAKE_FLAGS
 
@@ -191,7 +192,7 @@ if [ $MACOS ]; then
   JAR_LICENSE=$MINI_CLUSTER_SRCDIR/LICENSE-BINARY-JAR-OSX.txt
 fi
 cat $SOURCE_ROOT/LICENSE.txt \
-    $SOURCE_ROOT/thirdparty/LICENSE.txt \
+    $THIRDPARTY_DIR/LICENSE.txt \
     $JAR_LICENSE \
     > $ARTIFACT_NAME/LICENSE.txt
 

diff --git a/docs/support/scripts/make_site.sh b/docs/support/scripts/make_site.sh
@@ -73,8 +73,9 @@ if [ -z "$GIT_REMOTE" ]; then
   usage
 fi
 
+THIRDPARTY_DIR=${THIRDPARTY_DIR:-$SOURCE_ROOT/thirdparty}
 # Build Kudu thirdparty
-$SOURCE_ROOT/build-support/enable_devtoolset.sh $SOURCE_ROOT/thirdparty/build-if-necessary.sh
+$SOURCE_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_DIR/build-if-necessary.sh
 echo "Successfully built third-party dependencies."
 
 # Build the binaries so we can auto-generate the command-line references

diff --git a/src/kudu/clock/CMakeLists.txt b/src/kudu/clock/CMakeLists.txt
@@ -45,10 +45,10 @@ if (NOT NO_CHRONY)
   # Link the chrony binaries so that they can be found via
   # MiniChronyd::GetPath in MiniChronyd::Start.
   execute_process(COMMAND ln -nsf
-                  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/bin/chronyc"
+                  "${THIRDPARTY_DIR}/installed/common/bin/chronyc"
                   "${EXECUTABLE_OUTPUT_PATH}/chronyc")
   execute_process(COMMAND ln -nsf
-                  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/sbin/chronyd"
+                  "${THIRDPARTY_DIR}/installed/common/sbin/chronyd"
                   "${EXECUTABLE_OUTPUT_PATH}/chronyd")
 
   set(MINI_CHRONYD_SRCS test/mini_chronyd.cc)

diff --git a/src/kudu/hms/CMakeLists.txt b/src/kudu/hms/CMakeLists.txt
@@ -54,10 +54,10 @@ target_link_libraries(kudu_hms ${HMS_DEPS})
 # Link the home directories, so that they can be found via
 # test_util::FindHomeDir in MiniHms::Start.
 execute_process(COMMAND ln -nsf
-                "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hive"
+                "${THIRDPARTY_DIR}/installed/common/opt/hive"
                 "${EXECUTABLE_OUTPUT_PATH}/hive-home")
 execute_process(COMMAND ln -nsf
-                "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hadoop"
+                "${THIRDPARTY_DIR}/installed/common/opt/hadoop"
                 "${EXECUTABLE_OUTPUT_PATH}/hadoop-home")
 execute_process(COMMAND ln -nsf
                 "${JAVA_HOME}"

diff --git a/src/kudu/postgres/CMakeLists.txt b/src/kudu/postgres/CMakeLists.txt
@@ -20,16 +20,16 @@
 #######################################
 
 execute_process(COMMAND ln -nsf
-  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/bin"
+  "${THIRDPARTY_DIR}/installed/common/bin"
   "${EXECUTABLE_OUTPUT_PATH}/postgres")
 execute_process(COMMAND ln -nsf
-  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/lib"
+  "${THIRDPARTY_DIR}/installed/common/lib"
   "${EXECUTABLE_OUTPUT_PATH}/postgres-lib")
 execute_process(COMMAND ln -nsf
-  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/share/postgresql"
+  "${THIRDPARTY_DIR}/installed/common/share/postgresql"
   "${EXECUTABLE_OUTPUT_PATH}/postgres-share")
 execute_process(COMMAND ln -nsf
-  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/jdbc/postgresql.jar"
+  "${THIRDPARTY_DIR}/installed/common/opt/jdbc/postgresql.jar"
   "${EXECUTABLE_OUTPUT_PATH}/postgresql.jar")
 
 add_library(mini_postgres

diff --git a/src/kudu/ranger/CMakeLists.txt b/src/kudu/ranger/CMakeLists.txt
@@ -52,10 +52,10 @@ target_link_libraries(kudu_ranger ${RANGER_DEPS})
 #######################################
 
 execute_process(COMMAND ln -nsf
-  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/ranger"
+  "${THIRDPARTY_DIR}/installed/common/opt/ranger"
   "${EXECUTABLE_OUTPUT_PATH}/ranger-home")
 execute_process(COMMAND ln -nsf
-  "${CMAKE_SOURCE_DIR}/thirdparty/installed/common/opt/hadoop"
+  "${THIRDPARTY_DIR}/installed/common/opt/hadoop"
   "${EXECUTABLE_OUTPUT_PATH}/hadoop-home")
 execute_process(COMMAND ln -nsf
   "${JAVA_HOME}"

diff --git a/src/kudu/scripts/dump_breakpad_symbols.py b/src/kudu/scripts/dump_breakpad_symbols.py
@@ -56,7 +56,7 @@
 #   python dump_breakpad_symbols.py -d /tmp/syms -b kudu/build/debug
 #
 # * After, to process a minidump file, use the 'minidump_stackwalk' tool:
-#   $KUDU_HOME/thirdparty/installed/uninstrumented/bin/minidump_stackwalk \
+#   $THIRDPARTY_DIR/installed/uninstrumented/bin/minidump_stackwalk \
 #   /tmp/kudu-minidumps/kudu-tserver/03c0ee26-bfd1-cf3e-43fa49ca-1a6aae25.dmp /tmp/syms
 #
 # For more information, see the getting started docs at
@@ -86,6 +86,12 @@ def die(msg=''):
   sys.exit(1)
 
 
+def get_thirdparty_dir():
+  env = os.environ.copy()
+  return env['THIRDPARTY_DIR'] if env.has_key('THIRDPARTY_DIR') else \
+    os.path.join(ROOT, "thirdparty")
+
+
 def find_dump_syms_binary():
   """Locate the 'dump_syms' binary from Breakpad.
 
@@ -98,7 +104,8 @@ def find_dump_syms_binary():
     if not os.path.isdir(kudu_home):
       die('Could not find KUDU_HOME directory')
     # TODO: Use dump_syms_mac if on macOS.
-    dump_syms = os.path.join(kudu_home, 'thirdparty', 'installed', 'uninstrumented', 'bin', 'dump_syms')
+    dump_syms = os.path.join(get_thirdparty_dir(),
+                             'installed', 'uninstrumented', 'bin', 'dump_syms')
     if not os.path.isfile(dump_syms):
       die('Could not find dump_syms executable at %s' % dump_syms)
     return dump_syms