Supports offline compilation of Paddle third-party libraries (PaddleP…

…addle#53744) * optimize logsumexp in small data scale * fix * fix * add #pragma once * compile protobuf offline * add submodlu gflags * check_submodules * check_submodules * add_submodule protobuf * add_submodule_protobuf * add_submodule * add .gitmodules * add_submodules * fix_compiler error * support offline compile * support offline compile * support offline_compile * remove cub * remove brpc * support offline compile * support offline compile * canning patching on cryptopp * modify .gitigonre of cryptopp * test * offline compile * add_submodule zlib * modify .gitmodules * modify .gitmodules * fix setup.py bug * delete submodule cryptopp * fix windows compile bug * fix xxhash compile problem --------- Co-authored-by: Asthestarsfalll <[email protected]> Co-authored-by: Asthestarsfalll <[email protected]>
Xreki · May 17, 2023 · 734dc44 · 734dc44
1 parent 2cb2801
commit 734dc44
Show file tree

Hide file tree

Showing 29 changed files with 153 additions and 84 deletions.
diff --git a/.gitignore b/.gitignore
@@ -52,11 +52,11 @@ CMakeSettings.json
 Makefile
 .test_env/
 .cache/
-third_party/
+build/third_party/
 
 *~
 bazel-*
-third_party/
+build/third_party/
 
 build_*
 # clion workspace.

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,39 @@
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "third_party/gflags"]
+	path = third_party/gflags
+	url = https://github.com/gflags/gflags.git
+[submodule "third_party/gloo"]
+	path = third_party/gloo
+	url = https://github.com/ziyoujiyi/gloo.git
+[submodule "third_party/dlpack"]
+	path = third_party/dlpack
+	url = https://github.com/dmlc/dlpack.git
+[submodule "third_party/utf8proc"]
+	path = third_party/utf8proc
+	url = https://github.com/JuliaStrings/utf8proc.git
+[submodule "third_party/warpctc"]
+	path = third_party/warpctc
+	url = https://github.com/baidu-research/warp-ctc.git
+[submodule "third_party/warprnnt"]
+	path = third_party/warprnnt
+	url = https://github.com/PaddlePaddle/warp-transducer.git
+[submodule "third_party/xxhash"]
+	path = third_party/xxhash
+	url = https://github.com/Cyan4973/xxHash.git
+[submodule "third_party/eigen3"]
+	path = third_party/eigen3
+	url = https://gitlab.com/libeigen/eigen.git
+[submodule "third_party/leveldb"]
+	path = third_party/leveldb
+	url = https://github.com/google/leveldb
+[submodule "third_party/threadpool"]
+	path = third_party/threadpool
+	url = https://github.com/progschj/ThreadPool.git
+[submodule "third_party/zlib"]
+	path = third_party/zlib
+	url = https://github.com/madler/zlib.git
+[submodule "third_party/glog"]
+	path = third_party/glog
+	url = https://github.com/google/glog.git
diff --git a/cmake/external/dlpack.cmake b/cmake/external/dlpack.cmake
@@ -15,19 +15,15 @@
 include(ExternalProject)
 
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
-
-set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
-
 set(DLPACK_TAG v0.4)
-
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)
-include_directories(${DLPACK_INCLUDE_DIR})
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/dlpack)
+include_directories(${SOURCE_DIR}/include)
 
 ExternalProject_Add(
   extern_dlpack
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${DLPACK_REPOSITORY}
-  GIT_TAG ${DLPACK_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${DLPACK_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""

diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake
@@ -17,8 +17,8 @@ include(ExternalProject)
 # update eigen to the commit id f612df27 on 03/16/2021
 set(EIGEN_PREFIX_DIR ${THIRD_PARTY_PATH}/eigen3)
 set(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3/src/extern_eigen3)
-set(EIGEN_REPOSITORY https://gitlab.com/libeigen/eigen.git)
 set(EIGEN_TAG f612df273689a19d25b45ca4f8269463207c4fee)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/eigen3)
 
 if(WIN32)
   add_definitions(-DEIGEN_STRONG_INLINE=inline)
@@ -28,14 +28,12 @@ elseif(LINUX)
     # which will cause compiler error of using __host__ funciont
     # in __host__ __device__
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Meta.h native_src)
-    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/util/Meta.h
-         native_dst)
+    file(TO_NATIVE_PATH ${SOURCE_DIR}/Eigen/src/Core/util/Meta.h native_dst)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorReductionGpu.h
          native_src1)
-    file(
-      TO_NATIVE_PATH
-      ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
-      native_dst1)
+    file(TO_NATIVE_PATH
+         ${SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
+         native_dst1)
     set(EIGEN_PATCH_COMMAND cp ${native_src} ${native_dst} && cp ${native_src1}
                             ${native_dst1})
   endif()
@@ -54,18 +52,17 @@ if(CMAKE_COMPILER_IS_GNUCC)
     # See: [Why calling some `git` commands before `patch`?]
     set(EIGEN_PATCH_COMMAND
         git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
-        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
+        ${SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
   endif()
 endif()
 
-set(EIGEN_INCLUDE_DIR ${EIGEN_SOURCE_DIR})
+set(EIGEN_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${EIGEN_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_eigen3
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${EIGEN_REPOSITORY}
-  GIT_TAG ${EIGEN_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${EIGEN_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ${EIGEN_PATCH_COMMAND}

diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake
@@ -19,9 +19,8 @@ set(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
 set(GFLAGS_INCLUDE_DIR
     "${GFLAGS_INSTALL_DIR}/include"
     CACHE PATH "gflags include directory." FORCE)
-set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git)
 set(GFLAGS_TAG "v2.2.2")
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gflags)
 if(WIN32)
   set(GFLAGS_LIBRARIES
       "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib"
@@ -64,10 +63,9 @@ if(WITH_ARM_BRPC)
 else()
   ExternalProject_Add(
     extern_gflags
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GFLAGS_REPOSITORY}
-    GIT_TAG ${GFLAGS_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
     PREFIX ${GFLAGS_PREFIX_DIR}
+    SOURCE_DIR ${SOURCE_DIR}
     UPDATE_COMMAND ""
     BUILD_COMMAND ${BUILD_COMMAND}
     INSTALL_COMMAND ${INSTALL_COMMAND}

diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake
@@ -19,9 +19,8 @@ set(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
 set(GLOG_INCLUDE_DIR
     "${GLOG_INSTALL_DIR}/include"
     CACHE PATH "glog include directory." FORCE)
-set(GLOG_REPOSITORY ${GIT_URL}/google/glog.git)
 set(GLOG_TAG v0.4.0)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/glog)
 if(WIN32)
   set(GLOG_LIBRARIES
       "${GLOG_INSTALL_DIR}/lib/glog.lib"
@@ -67,8 +66,7 @@ else()
   ExternalProject_Add(
     extern_glog
     ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOG_REPOSITORY}
-    GIT_TAG ${GLOG_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
     DEPENDS gflags
     PREFIX ${GLOG_PREFIX_DIR}
     UPDATE_COMMAND ""

diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
@@ -25,12 +25,11 @@ set(GLOO_LIBRARY_DIR
     "${GLOO_INSTALL_DIR}/lib"
     CACHE PATH "gloo library directory." FORCE)
 # As we add extra features for gloo, we use the non-official repo
-set(GLOO_REPOSITORY ${GIT_URL}/ziyoujiyi/gloo.git)
 set(GLOO_TAG v0.0.3)
 set(GLOO_LIBRARIES
     "${GLOO_INSTALL_DIR}/lib/libgloo.a"
     CACHE FILEPATH "gloo library." FORCE)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gloo)
 set(GLOO_PATCH_COMMAND "")
 if(WITH_GPU)
   if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
@@ -39,7 +38,7 @@ if(WITH_GPU)
          native_dst)
     set(GLOO_PATCH_COMMAND
         git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
   endif()
 endif()
 
@@ -58,28 +57,27 @@ if(CMAKE_COMPILER_IS_GNUCC)
     # See: [Why calling some `git` commands before `patch`?]
     set(GLOO_PATCH_COMMAND
         git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})
 
 ExternalProject_Add(
   ${GLOO_PROJECT}
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${GLOO_REPOSITORY}
-  GIT_TAG ${GLOO_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX "${GLOO_PREFIX_DIR}"
   UPDATE_COMMAND ""
   PATCH_COMMAND ${GLOO_PATCH_COMMAND}
   CONFIGURE_COMMAND ""
   BUILD_COMMAND
-    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake ..
-    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND} --build . && mkdir
-    -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
+    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
+    ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND}
+    --build . && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
   INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
                   ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
-  COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/"
+  COMMAND ${CMAKE_COMMAND} -E copy_directory "${SOURCE_DIR}/gloo/"
           "${GLOO_INCLUDE_DIR}/gloo"
   BUILD_BYPRODUCTS ${GLOO_LIBRARIES})
 

diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake
@@ -188,6 +188,7 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
   set(PROTOBUF_SOURCE_DIR
       ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}/src/${TARGET_NAME})
   set(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME})
+  set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/${TARGET_DIR_NAME})
 
   set(${TARGET_NAME}_INCLUDE_DIR
       "${PROTOBUF_INSTALL_DIR}/include"
@@ -289,14 +290,13 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
   else()
     ExternalProject_Add(
       ${TARGET_NAME}
-      ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-      GIT_REPOSITORY ${PROTOBUF_REPOSITORY}
-      GIT_TAG ${PROTOBUF_TAG}
+      ${EXTERNAL_PROJECT_LOG_ARGS}
       PREFIX ${PROTOBUF_PREFIX_DIR}
+      SOURCE_DIR ${SOURCE_DIR}
       UPDATE_COMMAND ""
       DEPENDS zlib
       CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} ${PROTOBUF_SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
+        ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
         -G${CMAKE_GENERATOR} -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_SKIP_RPATH=ON
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}

diff --git a/cmake/external/threadpool.cmake b/cmake/external/threadpool.cmake
@@ -15,17 +15,15 @@
 include(ExternalProject)
 
 set(THREADPOOL_PREFIX_DIR ${THIRD_PARTY_PATH}/threadpool)
-set(THREADPOOL_REPOSITORY ${GIT_URL}/progschj/ThreadPool.git)
 set(THREADPOOL_TAG 9a42ec1329f259a5f4881a291db1dcb8f2ad9040)
-
-set(THREADPOOL_INCLUDE_DIR ${THIRD_PARTY_PATH}/threadpool/src/extern_threadpool)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/threadpool)
+set(THREADPOOL_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${THREADPOOL_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_threadpool
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${THREADPOOL_REPOSITORY}
-  GIT_TAG ${THREADPOOL_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${THREADPOOL_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""

diff --git a/cmake/external/utf8proc.cmake b/cmake/external/utf8proc.cmake
@@ -17,9 +17,8 @@ include(ExternalProject)
 set(UTF8PROC_PREFIX_DIR ${THIRD_PARTY_PATH}/utf8proc)
 set(UTF8PROC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/utf8proc)
 # As we add extra features for utf8proc, we use the non-official repo
-set(UTF8PROC_REPOSITORY ${GIT_URL}/JuliaStrings/utf8proc.git)
 set(UTF8PROC_TAG v2.6.1)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/utf8proc)
 if(WIN32)
   set(UTF8PROC_LIBRARIES "${UTF8PROC_INSTALL_DIR}/lib/utf8proc_static.lib")
   add_definitions(-DUTF8PROC_STATIC)
@@ -31,9 +30,8 @@ include_directories(${UTF8PROC_INSTALL_DIR}/include)
 
 ExternalProject_Add(
   extern_utf8proc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${UTF8PROC_REPOSITORY}
-  GIT_TAG ${UTF8PROC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${UTF8PROC_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}

diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake
@@ -22,9 +22,8 @@ set(WARPCTC_PREFIX_DIR ${THIRD_PARTY_PATH}/warpctc)
 set(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc)
 # in case of low internet speed
 #set(WARPCTC_REPOSITORY  https://gitee.com/tianjianhe/warp-ctc.git)
-set(WARPCTC_REPOSITORY ${GIT_URL}/baidu-research/warp-ctc.git)
 set(WARPCTC_TAG bdc2b4550453e0ef2d3b5190f9c6103a84eff184)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warpctc)
 set(WARPCTC_SOURCE_DIR ${THIRD_PARTY_PATH}/warpctc/src/extern_warpctc)
 set(WARPCTC_PATCH_COMMAND "")
 set(WARPCTC_CCBIN_OPTION "")
@@ -85,9 +84,8 @@ endif()
 
 ExternalProject_Add(
   extern_warpctc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPCTC_REPOSITORY}
-  GIT_TAG ${WARPCTC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPCTC_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ${WARPCTC_PATCH_COMMAND}

diff --git a/cmake/external/warprnnt.cmake b/cmake/external/warprnnt.cmake
@@ -20,9 +20,8 @@ endif()
 
 set(WARPRNNT_PREFIX_DIR ${THIRD_PARTY_PATH}/warprnnt)
 set(WARPRNNT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warprnnt)
-set(WARPRNNT_REPOSITORY ${GIT_URL}/PaddlePaddle/warp-transducer.git)
 set(WARPRNNT_TAG 7ea6bfe748779c245a0fcaa5dd9383826273eff2)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warprnnt)
 set(WARPRNNT_INCLUDE_DIR
     "${WARPRNNT_INSTALL_DIR}/include"
     CACHE PATH "Warp-rnnt Directory" FORCE)
@@ -70,9 +69,8 @@ else()
 endif()
 ExternalProject_Add(
   extern_warprnnt
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPRNNT_REPOSITORY}
-  GIT_TAG ${WARPRNNT_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPRNNT_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ""