Disable sccache when building NCCL (#8708)

* Disable sccache when building NCCL * Fix nccl CMakeLists.txt
sungin-h · Jun 22, 2018 · fd32cc6 · fd32cc6
1 parent 0750967
commit fd32cc6
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 3 deletions.
diff --git a/third_party/nccl/CMakeLists.txt b/third_party/nccl/CMakeLists.txt
@@ -1,6 +1,7 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8 FATAL_ERROR)
 CMAKE_POLICY(VERSION 2.8)
 
+include(CMakeInitializeConfigs)
 IF(NOT CUDA_FOUND)
   FIND_PACKAGE(CUDA 7.0 REQUIRED)
 ENDIF()
@@ -9,7 +10,7 @@ include("${CMAKE_UTILS_PATH}")
 torch_cuda_get_nvcc_gencode_flag(NVCC_GENCODE)
 string (REPLACE ";" " " NVCC_GENCODE "${NVCC_GENCODE}")
 string (REPLACE "-gencode " "-gencode=" NVCC_GENCODE "${NVCC_GENCODE}")
-message(INFO "Set NVCC_GENCODE for building NCCL: ${NVCC_GENCODE}")
+message(STATUS "Set NVCC_GENCODE for building NCCL: ${NVCC_GENCODE}")
 
 ADD_CUSTOM_COMMAND(
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}

diff --git a/tools/build_pytorch_libs.sh b/tools/build_pytorch_libs.sh
@@ -144,7 +144,7 @@ function build() {
   # TODO: The *_LIBRARIES cmake variables should eventually be
   # deprecated because we are using .cmake files to handle finding
   # installed libraries instead
-  ${CMAKE_VERSION} ../../$1 -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/FindCUDA" \
+  ${CMAKE_VERSION} ../../$1 -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/Modules_CUDA_fix" \
               ${CMAKE_GENERATOR} \
               -DTorch_FOUND="1" \
               -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
@@ -194,10 +194,22 @@ function build() {
   fi
 }
 
+function path_remove {
+  # Delete path by parts so we can never accidentally remove sub paths
+  PATH=${PATH//":$1:"/":"} # delete any instances in the middle
+  PATH=${PATH/#"$1:"/} # delete any instance at the beginning
+  PATH=${PATH/%":$1"/} # delete any instance in the at the end
+}
+
 function build_nccl() {
+  # FIXME: sccache doesn't work when building NCCL
+  path_remove /opt/cache/bin
+  export CUDA_NVCC_EXECUTABLE_SAVED=$CUDA_NVCC_EXECUTABLE
+  export CUDA_NVCC_EXECUTABLE=$(which nvcc)
+
   mkdir -p build/nccl
   pushd build/nccl
-  ${CMAKE_VERSION} ../../nccl -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/FindCUDA" \
+  ${CMAKE_VERSION} ../../nccl -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/Modules_CUDA_fix" \
               ${CMAKE_GENERATOR} \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
@@ -212,6 +224,10 @@ function build_nccl() {
     ln -s "${INSTALL_DIR}/lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so"
   fi
   popd
+
+  # FIXME: sccache doesn't work when building NCCL
+  export CUDA_NVCC_EXECUTABLE=$CUDA_NVCC_EXECUTABLE_SAVED
+  export PATH=/opt/cache/bin:$PATH
 }
 
 # purposefully not using build() because we need Caffe2 to build the same