Upgrade to the latest boost version (1.80) (#2857)

Upgrade yaml-cpp to a newer version (0.7.0) as it used a non-compatible boost version and now it's a boost free version.
Upgrade libthrift to v0.13.0
Upgrade libjaegertracing-cpp to v0.9.0

CMakeLists.txt

@@ -1,17 +1,15 @@
-cmake_minimum_required(VERSION 3.2)
+cmake_minimum_required(VERSION 3.24)
 project(concord-bft VERSION 0.1.0.0 LANGUAGES CXX)
 
-#
-# C++ options
-#   TODO: change to set_target_properties?
-#   https://crascit.com/2015/03/28/enabling-cxx11-in-cmake/
-#
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(SLEEP_FOR_DBG FALSE)
-
-set(MIN_BOOST_VERSION 1.64)
+# targets with generic names like "format" may already exist in an imported libs
+cmake_policy(SET CMP0002 OLD)
+set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
+set(MIN_BOOST_VERSION 1.80)
+set(YAML_CPP_VERSION 0.7.0)
 
 # Default to debug builds
 # Release builds can be enabled by running cmake with -DCMAKE_BUILD_TYPE=Release
@@ -43,7 +41,7 @@ option(USE_JSON "Enable use of JSON library" ON)
 option(USE_HTTPLIB "Enable use of httplib" ON)
 option(USE_GRPC "Enable GRPC and Protobuf" ON)
 option(USE_OPENSSL "Enable use of OpenSSL" ON)
-option(BUILD_THIRDPARTY "Wheter to build third party librarie or use preinstalled ones" OFF)
+option(BUILD_THIRDPARTY "Whether to build third party libraries or use preinstalled ones" OFF)
 option(CODECOVERAGE "Enable Code Coverage Metrics in Clang" OFF)
 option(ENABLE_RESTART_RECOVERY_TESTS "Enable tests for restart recovery" OFF)
 option(BUILD_UTT "Build UTT library" ON)

Makefile

@@ -1,6 +1,6 @@
 CONCORD_BFT_DOCKER_REPO?=concordbft/
 CONCORD_BFT_DOCKER_IMAGE?=concord-bft
-CONCORD_BFT_DOCKER_IMAGE_VERSION?=0.48
+CONCORD_BFT_DOCKER_IMAGE_VERSION?=0.49
 CONCORD_BFT_DOCKER_CONTAINER?=concord-bft
 
 CONCORD_BFT_DOCKERFILE?=Dockerfile

bftengine/CMakeLists.txt

@@ -2,6 +2,7 @@ project (bftengine LANGUAGES CXX)
 
 add_subdirectory(src/preprocessor)
 find_package(CMFC REQUIRED)
+find_package(Boost ${MIN_BOOST_VERSION})
 add_subdirectory("cmf")
 
 set(corebft_source_files
@@ -100,6 +101,7 @@ target_include_directories(corebft PRIVATE src/bftengine)
 target_include_directories(corebft PRIVATE src/preprocessor)
 target_include_directories(corebft PUBLIC ${perf_include}/performance/include)
 target_include_directories(corebft PUBLIC tests/mocks)
+target_include_directories(corebft PUBLIC ${Boost_INCLUDE_DIR})
 target_link_libraries(corebft PUBLIC
   threshsign
   Threads::Threads

client/clientservice/CMakeLists.txt

@@ -1,13 +1,13 @@
 find_package(GRPC REQUIRED)
 find_package(Boost ${MIN_BOOST_VERSION} COMPONENTS program_options REQUIRED)
 find_package(jaegertracing REQUIRED)
-
+find_package(yaml-cpp ${YAML_CPP_VERSION} REQUIRED)
 # All source files except for main.cpp are part of the library
 file(GLOB CLIENTSERVICE_SRC "src/*.cpp")
 list(FILTER CLIENTSERVICE_SRC EXCLUDE REGEX "src/main.cpp")
 
 add_library(clientservice-lib ${CLIENTSERVICE_SRC})
-target_include_directories(clientservice-lib PUBLIC include)
+target_include_directories(clientservice-lib PUBLIC include ${Boost_INCLUDE_DIR})
 target_link_libraries(clientservice-lib PUBLIC
   clientservice-proto
   concordclient

client/clientservice/src/main.cpp

@@ -15,7 +15,7 @@
 #include <vector>
 #include <chrono>
 #include <boost/program_options.hpp>
-#include <boost/asio/signal_set.hpp>
+#include <boost/asio.hpp>
 #include <yaml-cpp/yaml.h>
 
 #include "client/clientservice/client_service.hpp"

client/thin-replica-client/src/trc_hash.cpp

@@ -16,7 +16,7 @@
 #include <map>
 #include <set>
 
-#include "boost/detail/endian.hpp"
+#include "endianness.hpp"
 #include "assertUtils.hpp"
 
 using com::vmware::concord::thin_replica::Data;
@@ -52,19 +52,19 @@ static string hashUpdateFromEntryHashes(uint64_t block_id, const map<string, str
   string concatenated_entry_hashes;
   concatenated_entry_hashes.reserve(sizeof(block_id) + 2 * entry_hashes.size() * kThinReplicaHashLength);
 
-#ifdef BOOST_LITTLE_ENDIAN
+#ifdef CONCORD_LITTLE_ENDIAN
   concatenated_entry_hashes.append(reinterpret_cast<const char*>(&block_id), sizeof(block_id));
-#else  // BOOST_LITTLE_ENDIAN not defined in this case
-#ifndef BOOST_BIG_ENDIAN
+#else  // CONCORD_LITTLE_ENDIAN not defined in this case
+#ifndef CONCORD_BIG_ENDIAN
   static_assert(false,
                 "Cannot determine endianness (needed for Thin Replica "
                 "mechanism hash function).");
-#endif  // BOOST_BIG_ENDIAN defined
+#endif  // CONCORD_BIG_ENDIAN defined
   const char* block_id_as_bytes = reinterpret_cast<const char*>(&block_id);
   for (size_t i = 1; i <= sizeof(block_id); ++i) {
     concatenated_entry_hashes.append((block_id_as_bytes + (sizeof(block_id) - i)), 1);
   }
-#endif  // if BOOST_LITTLE_ENDIAN defined/else
+#endif  // if CONCORD_LITTLE_ENDIAN defined/else
 
   for (const auto& kvp_hashes : entry_hashes) {
     concatenated_entry_hashes.append(kvp_hashes.first);
@@ -78,19 +78,19 @@ static string hashUpdateFromEntryHashes(uint64_t id, const set<string>& entry_ha
   string concatenated_entry_hashes;
   concatenated_entry_hashes.reserve(sizeof(id) + entry_hashes.size() * kThinReplicaHashLength);
 
-#ifdef BOOST_LITTLE_ENDIAN
+#ifdef CONCORD_LITTLE_ENDIAN
   concatenated_entry_hashes.append(reinterpret_cast<const char*>(&(id)), sizeof(id));
-#else  // BOOST_LITTLE_ENDIAN not defined in this case
-#ifndef BOOST_BIG_ENDIAN
+#else  // CONCORD_LITTLE_ENDIAN not defined in this case
+#ifndef CONCORD_BIG_ENDIAN
   static_assert(false,
                 "Cannot determine endianness (needed for Thin Replica "
                 "mechanism hash function).");
-#endif  // BOOST_BIG_ENDIAN defined
+#endif  // CONCORD_BIG_ENDIAN defined
   const char* id_as_bytes = reinterpret_cast<const char*>(&(id));
   for (size_t i = 1; i <= sizeof(id); ++i) {
     concatenated_entry_hashes.append((id_as_bytes + (sizeof(id) - i)), 1);
   }
-#endif  // if BOOST_LITTLE_ENDIAN defined/else
+#endif  // if CONCORD_LITTLE_ENDIAN defined/else
 
   for (const auto& hash : entry_hashes) {
     concatenated_entry_hashes.append(hash);

communication/CMakeLists.txt

@@ -31,23 +31,14 @@ if(BUILD_COMM_TCP_TLS)
 endif()
 
 target_link_libraries(bftcommunication PRIVATE stdc++fs)
-set(Boost_USE_STATIC_LIBS OFF) # find all kind of libs
-set(Boost_USE_MULTITHREADED ON)
-set(Boost_USE_STATIC_RUNTIME OFF)
-find_package(Boost ${MIN_BOOST_VERSION} COMPONENTS system REQUIRED)
-target_include_directories(bftcommunication PUBLIC ${Boost_INCLUDE_DIRS})
-target_link_libraries(bftcommunication PUBLIC ${Boost_LIBRARIES})
 if(BUILD_COMM_TCP_PLAIN)
     target_compile_definitions(bftcommunication PUBLIC USE_COMM_PLAIN_TCP)
 elseif(BUILD_COMM_TCP_TLS)
     if(NOT USE_OPENSSL)
         message(FATAL_ERROR "-DUSE_OPENSSL should be specified if building with -DBUILD_COMM_TCP_TLS")
     endif()
-    if(NOT BUILD_THIRDPARTY)
-        find_package(OpenSSL REQUIRED)
-    endif()
+    find_package(OpenSSL REQUIRED)
     target_link_libraries(bftcommunication        PUBLIC OpenSSL::SSL diagnostics)
-
     target_compile_definitions(bftcommunication        PUBLIC USE_COMM_TLS_TCP)
 
 endif()

diagnostics/CMakeLists.txt

@@ -3,12 +3,9 @@ project(libdiagnostics VERSION 0.1.0.0 LANGUAGES CXX)
 
 add_library(diagnostics src/status_handlers.cpp src/performance_handler.cpp)
 
-if(BUILD_THIRDPARTY)
-    add_dependencies(diagnostics hdrhistogram)
-else()
-    find_path(HDR_HISTOGRAM_INCLUDE_DIR "hdr_interval_recorder.h" HINTS /usr/local/include/hdr REQUIRED)
-    find_library(HDR_HISTOGRAM_LIBRARY hdr_histogram HINTS /usr/local/lib REQUIRED)
-endif()
+
+find_path(HDR_HISTOGRAM_INCLUDE_DIR "hdr_interval_recorder.h" HINTS /usr/local/include/hdr REQUIRED)
+find_library(HDR_HISTOGRAM_LIBRARY hdr_histogram HINTS /usr/local/lib REQUIRED)
 
 set_property(TARGET diagnostics PROPERTY POSITION_INDEPENDENT_CODE ON)
 target_include_directories(diagnostics PUBLIC include ${HDR_HISTOGRAM_INCLUDE_DIR})

examples/client/CMakeLists.txt

@@ -1,5 +1,7 @@
 project (test_client LANGUAGES CXX)
 
+find_package(yaml-cpp ${YAML_CPP_VERSION} REQUIRED)
+
 set(test_client_sources
     main.cpp
     src/SetupClient.cpp

examples/replica/CMakeLists.txt

@@ -1,5 +1,7 @@
 project (test_replica LANGUAGES CXX)
 
+find_package(yaml-cpp ${YAML_CPP_VERSION} REQUIRED)
+
 set(test_replica_source 
         main.cpp
         src/KVCommandHandler.cpp

install_deps.sh

@@ -6,7 +6,7 @@
 # If you prefer working w/o docker at your dev station just run the script with sudo.
 # If you need to add any tool or dependency this is the right place to do it.
 # Please bear in mind that we want to keep our list of dependencies as small as possible.
-set -e
+set -ex
 
 APT_GET_FLAGS="-y --no-install-recommends"
 WGET_FLAGS="--https-only"
@@ -18,17 +18,20 @@ install_build_tools() {
     apt-get update && apt-get ${APT_GET_FLAGS} install \
         autoconf \
         automake \
-        libtool \
+        bison \
         build-essential \
         clang-9 \
         clang-format-10 \
         clang-tidy-10 \
         curl \
+        flex \
         gdb \
         gdbserver \
         git \
+        iproute2 \
         iptables \
         less \
+        libtool \
         llvm-9 \
         lzip \
         net-tools \
@@ -40,7 +43,6 @@ install_build_tools() {
         python3-dev \
         sudo \
         vim \
-        iproute2 \
         wget
 
     update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-9/bin/clang 100
@@ -57,15 +59,11 @@ install_build_tools() {
 # Install 3rd parties
 install_third_party_libraries() {
     apt-get ${APT_GET_FLAGS} install \
-        libboost-system1.65-dev \
-        libboost-program-options1.65-dev \
-        libboost1.65-dev \
         libbz2-dev \
         liblz4-dev \
         libs3-dev \
         libsnappy-dev \
         libssl-dev \
-        libyaml-cpp-dev \
         libz-dev \
         libzstd-dev
 
@@ -86,8 +84,31 @@ install_third_party_libraries() {
 # Build 3rd parties
 install_cmake() {
     wget ${WGET_FLAGS} -O cmake-linux.sh \
-        https://github.com/Kitware/CMake/releases/download/v3.20.2/cmake-3.20.2-linux-x86_64.sh && \
-        sh cmake-linux.sh -- --skip-license --prefix=/usr
+    https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh &&\
+    sh cmake-linux.sh -- --skip-license --prefix=/usr
+}
+
+install_boost() {
+    cd ${HOME}
+    git clone --recurse-submodules --depth=1 --single-branch --branch=boost-1.80.0 https://github.com/boostorg/boost.git
+    cd boost
+    mkdir build
+    cd build
+    cmake -DBOOST_INCLUDE_LIBRARIES="program_options;thread;locale;asio;lockfree;bimap" ..
+    make install
+    cd ${HOME}
+    rm -rf boost
+}
+
+install_yaml() {
+    cd ${HOME}
+    git clone -b yaml-cpp-0.7.0 --single-branch  https://github.com/jbeder/yaml-cpp.git
+    mkdir yaml-cpp/build
+    cd yaml-cpp/build
+    cmake -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF -DYAML_BUILD_SHARED_LIBS=OFF ..
+    make -j$(nproc) install
+    cd ${HOME}
+    rm -rf xvzf yaml-cpp
 }
 
 install_HdrHistogram_lib() {
@@ -369,42 +390,39 @@ install_httplib() {
 }
 
 # Thrift is the protocol used by Jaeger to export metrics
+#depends: boost
 install_thrift_lib(){
 cd $HOME
-wget ${WGET_FLAGS} https://archive.apache.org/dist/thrift/0.11.0/thrift-0.11.0.tar.gz && \
-    tar xzf thrift-0.11.0.tar.gz && \
-    cd thrift-0.11.0 && \
-    ./configure CXXFLAGS='-g -O2' \
+  git clone -b v0.13.0 https://github.com/apache/thrift.git
+  cd thrift
+  ./bootstrap.sh
+  ./configure CXXFLAGS='-g -O2' \
       --without-python --enable-static --disable-shared \
-      --disable-tests --disable-tutorial --disable-coverage && \
-    make -j$(nproc) install && \
-    cd ${HOME} && \
-    rm -r thrift-0.11.0 thrift-0.11.0.tar.gz
+      --disable-tests --disable-tutorial --disable-coverage
+  make -j$(nproc) install
+  cd ${HOME}
+  rm -r thrift
 }
 
 install_jaegertracing_cpp_lib(){
   # TODO: Upgrade to opentelemetry-cpp
   # Tracing via Jaeger and Thrift protocol
   # Copy FindThrift.cmake because installing Thrift does not include a CMake definition
+
+  #depends on opentracing, thrift, nlohmann, httplib
   cd $HOME
-  git clone -b v0.7.0 --depth 1 https://github.com/jaegertracing/jaeger-client-cpp && \
-      cd jaeger-client-cpp && \
-      mkdir build && \
-      cd build && \
-      cmake -DHUNTER_ENABLED=OFF -DHUNTER_BUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF \
+  git clone -b v0.9.0 --depth 1 https://github.com/jaegertracing/jaeger-client-cpp
+  cd jaeger-client-cpp && mkdir build && cd build
+  cp ../cmake/Findthrift.cmake /usr/share/cmake-3.24/Modules/
+  cmake     -DHUNTER_ENABLED=OFF -DHUNTER_BUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF \
             -DBUILD_SHARED_LIBS=OFF -DJAEGERTRACING_BUILD_EXAMPLES=OFF \
             -DJAEGERTRACING_PLUGIN=OFF -DJAEGERTRACING_COVERAGE=OFF \
             -DJAEGERTRACING_BUILD_CROSSDOCK=OFF -DJAEGERTRACING_WITH_YAML_CPP=OFF \
-            .. && \
-      make -j$(nproc) install && \
-      cp ../cmake/Findthrift.cmake /usr/share/cmake-3.20/Modules/ && \
-      cd ${HOME} && \
-      rm -r jaeger-client-cpp
-
-  # Jaeger really wants to find BoostConfig.cmake, not FindBoost.cmake.
-  # This wasn't introduced until boost 1.70.
-  # Jaegertracing.cmake finds FindBoost.cmake first anyways.
-  # The following sed just removes the search for BoostConfig.cmake.
+            ..
+  make -j$(nproc) install
+  cd ${HOME}
+  rm -r jaeger-client-cpp
+
   sed -i '/boost_components/d' /usr/local/lib/cmake/jaegertracing/jaegertracingConfig.cmake
 }
 
@@ -442,6 +460,8 @@ install_ccache(){
 install_build_tools
 install_third_party_libraries
 install_cmake
+install_boost
+install_yaml
 install_HdrHistogram_lib
 install_log4cpp_lib
 install_googletest

kvbc/CMakeLists.txt

@@ -72,9 +72,8 @@ target_link_libraries(kvbc PUBLIC concord_block_update concord-kvbc-proto)
 
 target_include_directories(kvbc PUBLIC ${PROJECT_SOURCE_DIR} include util)
 
-if(NOT BUILD_THIRDPARTY)
-    find_package(OpenSSL REQUIRED)
-endif()
+find_package(OpenSSL REQUIRED)
+
 target_link_libraries(kvbc PRIVATE OpenSSL::Crypto)
 
 if (BUILD_TESTING)

kvbc/benchmark/kvbcbench/CMakeLists.txt

@@ -4,7 +4,7 @@ add_executable(kvbcbench main.cpp)
 target_link_libraries(kvbcbench PUBLIC
     kvbc
     util
-    ${Boost_LIBRARIES}
+    Boost::program_options
 )
 file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/kvbcbench_rocksdb_opts.ini
        DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)

kvbc/benchmark/state_snapshot_benchmarks/CMakeLists.txt

@@ -4,5 +4,5 @@ add_executable(hash_state_benchmark hash_state_benchmark.cpp)
 target_link_libraries(hash_state_benchmark PUBLIC
     kvbc
     util
-    ${Boost_LIBRARIES}
+    Boost::program_options
 )