Fix cmake build failure with gflags (facebook#8324)

Summary: - Fix cmake build failure with gflags. - Add CI tests for both gflags 2.1 and 2.2. - Fix ctest config with gtest. - Add CI to run test with ctest. One benefit of ctest is it support timeout, it's set to 5min in our CI, so we will know which test is hang. Pull Request resolved: facebook#8324 Test Plan: CI pass Reviewed By: ajkr Differential Revision: D28762517 Pulled By: jay-zhuang fbshipit-source-id: 09063c5af5f9f33abfcdeb48593acbd9826cd199
tianxiaogu · Jun 1, 2021 · eda83ea · eda83ea
1 parent ab718b4
commit eda83ea
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 40 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -49,6 +49,9 @@ commands:
             echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
             echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
             echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
+            echo "export GTEST_COLOR=1" >> $BASH_ENV
+            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
+            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
 
   pre-steps-macos:
       steps:
@@ -88,6 +91,14 @@ commands:
           command: |
             sudo apt-get update -y && sudo apt-get install -y libgflags-dev
 
+  upgrade-cmake:
+    steps:
+      - run:
+          name: Upgrade cmake
+          command: |
+            sudo apt remove --purge cmake
+            sudo snap install cmake --classic
+
   install-gflags-on-macos:
     steps:
       - run:
@@ -140,12 +151,12 @@ jobs:
       - install-cmake-on-macos
       - install-gflags-on-macos
       - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
+      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain
       - post-steps
 
   build-linux:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -155,7 +166,7 @@ jobs:
 
   build-linux-mem-env:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -165,7 +176,7 @@ jobs:
 
   build-linux-encrypted-env:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -175,7 +186,7 @@ jobs:
 
   build-linux-shared_lib-alt_namespace-status_checked:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -185,7 +196,7 @@ jobs:
 
   build-linux-release:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: large
     steps:
       - checkout # check out the code in the project directory
@@ -212,7 +223,7 @@ jobs:
 
   build-linux-lite:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -222,7 +233,7 @@ jobs:
 
   build-linux-lite-release:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: large
     steps:
       - checkout # check out the code in the project directory
@@ -235,7 +246,7 @@ jobs:
 
   build-linux-clang-no_test_run:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: xlarge
     steps:
       - checkout # check out the code in the project directory
@@ -245,7 +256,7 @@ jobs:
 
   build-linux-clang10-asan:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -256,7 +267,7 @@ jobs:
 
   build-linux-clang10-mini-tsan:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -267,7 +278,7 @@ jobs:
 
   build-linux-clang10-ubsan:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -278,7 +289,7 @@ jobs:
 
   build-linux-clang10-clang-analyze:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -288,13 +299,25 @@ jobs:
       - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
       - post-steps
 
-  build-linux-cmake-no_test_run:
+  build-linux-cmake:
     machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
+      image: ubuntu-1604:202104-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - install-gflags
+      - upgrade-cmake
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
+      - post-steps
+
+  build-linux-cmake-ubuntu-20:
+    machine:
+      image: ubuntu-2004:202104-01
+    resource_class: 2xlarge
     steps:
       - checkout # check out the code in the project directory
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain
+      - install-gflags
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
       - post-steps
 
   build-linux-unity:
@@ -309,7 +332,7 @@ jobs:
 
   build-linux-gcc-4_8-no_test_run:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: large
     steps:
       - pre-steps
@@ -417,7 +440,7 @@ jobs:
 
   build-linux-java:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: large
     environment:
       JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
@@ -441,7 +464,7 @@ jobs:
 
   build-linux-java-static:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: large
     environment:
       JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
@@ -512,7 +535,7 @@ jobs:
 
   build-examples:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: large
     steps:
       - pre-steps
@@ -525,7 +548,7 @@ jobs:
 
   build-cmake-mingw:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -548,7 +571,7 @@ jobs:
 
   build-linux-non-shm:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     parameters:
       start_test:
@@ -604,6 +627,7 @@ jobs:
       JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
     steps:
       - pre-steps
+      - install-gflags
       - run:
           name: "Set Java Environment"
           command: |
@@ -616,21 +640,21 @@ jobs:
           command: |
             mkdir build
             cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
+            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
             make -j4
       - run:
           name: "Build Java with cmake"
           command: |
             rm -rf build
             mkdir build
             cd build
-            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=0 ..
+            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
             make -j4 rocksdb rocksdbjni
       - post-steps
 
   build-format-compatible:
     machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
     resource_class: 2xlarge
     steps:
       - pre-steps
@@ -650,6 +674,10 @@ workflows:
   build-linux:
     jobs:
       - build-linux
+  build-linux-cmake:
+    jobs:
+      - build-linux-cmake
+      - build-linux-cmake-ubuntu-20
   build-linux-mem-env:
     jobs:
       - build-linux-mem-env
@@ -733,7 +761,6 @@ workflows:
   build-linux-compilers-no_test_run:
     jobs:
       - build-linux-clang-no_test_run
-      - build-linux-cmake-no_test_run
       - build-linux-gcc-4_8-no_test_run
       - build-linux-gcc-8-no_test_run
       - build-linux-gcc-9-no_test_run

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -32,10 +32,11 @@
 # 3. cmake ..
 # 4. make -j
 
-cmake_minimum_required(VERSION 3.5.1)
+cmake_minimum_required(VERSION 3.10)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
 include(ReadVersion)
+include(GoogleTest)
 get_rocksdb_version(rocksdb_VERSION)
 project(rocksdb
   VERSION ${rocksdb_VERSION}
@@ -71,6 +72,12 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files,
 if (WITH_WINDOWS_UTF8_FILENAMES)
   add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
 endif()
+
+if ($ENV{CIRCLECI})
+  message(STATUS "Build for CircieCI env, a few tests may be disabled")
+  add_definitions(-DCIRCLECI)
+endif()
+
 # third-party/folly is only validated to work on Linux and Windows for now.
 # So only turn it on there by default.
 if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
@@ -123,11 +130,11 @@ else()
         set(GFLAGS_LIB ${GFLAGS_TARGET})
       else()
         # Config with GFLAGS_LIBRARIES available since gflags 2.1.0
-        set(GFLAGS_LIB ${GFLAGS_LIBRARIES})
+        set(GFLAGS_LIB ${gflags_LIBRARIES})
       endif()
     else()
       find_package(gflags REQUIRED)
-        set(GFLAGS_LIB gflags::gflags)
+      set(GFLAGS_LIB gflags::gflags)
     endif()
     include_directories(${GFLAGS_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB})
@@ -211,7 +218,7 @@ if(MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
 else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
   if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes")
@@ -1321,21 +1328,21 @@ if(WITH_TESTS)
 
   foreach(sourcefile ${TESTS})
       get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
+      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+      set_target_properties(${exename}${ARTIFACT_SUFFIX}
         PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
         OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
       )
-      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
+      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
       if(NOT "${exename}" MATCHES "db_sanity_test")
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
+        gtest_discover_tests(${exename})
+        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
       endif()
       if("${exename}" MATCHES "env_librados_test")
         # env_librados_test.cc uses librados directly
-        target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} rados)
+        target_link_libraries(${exename}${ARTIFACT_SUFFIX} rados)
       endif()
   endforeach(sourcefile ${TESTS})
 

diff --git a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
@@ -9,7 +9,7 @@
 int main() {
   fprintf(stderr,
           "Please install gflags to run block_cache_trace_analyzer_test\n");
-  return 1;
+  return 0;
 }
 #else
 

diff --git a/tools/io_tracer_parser_test.cc b/tools/io_tracer_parser_test.cc
@@ -9,7 +9,7 @@
 #include <cstdio>
 int main() {
   fprintf(stderr, "Please install gflags to run io_tracer_parser_test\n");
-  return 1;
+  return 0;
 }
 #else
 

diff --git a/tools/trace_analyzer_test.cc b/tools/trace_analyzer_test.cc
@@ -12,7 +12,7 @@
 #include <cstdio>
 int main() {
   fprintf(stderr, "Please install gflags to run trace_analyzer test\n");
-  return 1;
+  return 0;
 }
 #else