Introduce a reserved pages client for pruning (vmware#1637)

Provide persistence of pruning agreement and progress information in
reserved pages. Pseudo code for how it can be used is provided as an
inline comment.

Introduce the `ReservedPagesMock` class and reuse it in ccron and
pruning reserved pages client unit tests. In order to make it work, add
the static `numberOfReservedPagesForClient()` method and make the
`my_offset()` and `calc_my_offset()` methods static instead of const in
`ResPagesClient`. That allows for a simpler init procedure in unit tests
as we can now call them without an instance.

Move the `INITIAL_GENESIS_BLOCK_ID` constant to kv_types.hpp and
reuse it across the codebase.

Add an unit test to verify the pruning reserved pages client's
functionality.

* Make `ReservedPagesMock` a friend of the client

Also, make `my_offset()` private.

* Address review comments

 * numberOfReservedPagesForClient()
 * use composition in ReservedPagesClient

bftengine/CMakeLists.txt

@@ -104,6 +104,7 @@ target_include_directories(corebft PRIVATE src/bftengine)
 target_include_directories(corebft PRIVATE src/preprocessor)
 target_include_directories(corebft PUBLIC ${CRYPTOPP_INCLUDE_DIRS})
 target_include_directories(corebft PUBLIC ${perf_include}/performance/include)
+target_include_directories(corebft PUBLIC tests/mocks)
 target_link_libraries(corebft PUBLIC
   threshsign
   Threads::Threads

bftengine/include/bftengine/ReservedPagesClient.hpp

@@ -23,6 +23,11 @@
 
 namespace bftEngine {
 
+namespace test {
+template <typename, uint32_t>
+class ReservedPagesMock;
+}
+
 class ReservedPagesClientBase {
  public:
   static uint32_t totalNumberOfPages() {
@@ -81,6 +86,16 @@ class ResPagesClient : public ReservedPagesClientBase, public IReservedPages {
     }
   }
 
+  static uint32_t numberOfReservedPagesForClient() {
+    auto& reg = registry();
+    if (auto it = reg.find(std::type_index(typeid(T))); it != reg.end()) {
+      return it->second;
+    } else {
+      std::cerr << __PRETTY_FUNCTION__ << " BUG: not registered" << std::endl;
+      std::terminate();
+    }
+  }
+
   uint32_t numberOfReservedPages() const override { return res_pages_->numberOfReservedPages(); }
   uint32_t sizeOfReservedPage() const override { return res_pages_->sizeOfReservedPage(); }
   bool loadReservedPage(uint32_t reservedPageId, uint32_t copyLength, char* outReservedPage) const override {
@@ -93,15 +108,18 @@ class ResPagesClient : public ReservedPagesClientBase, public IReservedPages {
     res_pages_->zeroReservedPage(my_offset() + reservedPageId);
   }
 
+ private:
+  template <typename, uint32_t>
+  friend class bftEngine::test::ReservedPagesMock;
+
   /** is called when calculating absolute pageId */
-  uint32_t my_offset() const {
+  static uint32_t my_offset() {
     static uint32_t offset_ = calc_my_offset();
     return offset_;
   }
 
- private:
   // is done once per client
-  uint32_t calc_my_offset() const {
+  static uint32_t calc_my_offset() {
     uint32_t offset = 0;
     for (auto& it : registry()) {
       if (it.first == std::type_index(typeid(T))) {

bftengine/tests/mocks/ReservedPagesMock.hpp

@@ -0,0 +1,69 @@
+// Concord
+//
+// Copyright (c) 2021 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the
+// "License").  You may not use this product except in compliance with the
+// Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright
+// notices and license terms. Your use of these subcomponents is subject to the
+// terms and conditions of the subcomponent's license, as noted in the LICENSE
+// file.
+
+#pragma once
+
+#include "IReservedPages.hpp"
+
+#include <algorithm>
+#include <cstring>
+#include <map>
+#include <string>
+
+namespace bftEngine::test {
+
+template <typename ReservedPagesClient, uint32_t kSizeOfReservedPage = 4096>
+class ReservedPagesMock : public bftEngine::IReservedPages {
+ public:
+  uint32_t numberOfReservedPages() const override { return ReservedPagesClient::numberOfReservedPagesForClient(); }
+
+  uint32_t sizeOfReservedPage() const override { return kSizeOfReservedPage; }
+
+  bool loadReservedPage(uint32_t page_id, uint32_t size, char* data) const override {
+    auto it = pages_.find(page_id);
+    if (it != pages_.cend()) {
+      std::memcpy(data, it->second.data(), std::min<std::size_t>(size, it->second.size()));
+      return true;
+    }
+    return false;
+  }
+
+  void saveReservedPage(uint32_t page_id, uint32_t size, const char* data) override {
+    pages_[page_id].assign(data, std::min<std::size_t>(size, sizeOfReservedPage()));
+  }
+
+  void zeroReservedPage(uint32_t page_id) override {
+    auto it = pages_.find(page_id);
+    if (it != pages_.cend()) {
+      for (auto i = 0ul; i < it->second.size(); ++i) {
+        it->second[i] = '\0';
+      }
+    }
+  }
+
+  bool isReservedPageZeroed(uint32_t page_id) const {
+    auto it = pages_.find(ReservedPagesClient::my_offset() + page_id);
+    if (it == pages_.cend()) {
+      return false;
+    }
+    return (it->second.find_first_not_of('\0') == std::string::npos);
+  }
+
+  const std::map<uint32_t, std::string>& pages() const { return pages_; }
+
+ private:
+  // reserved page ID -> contents
+  std::map<uint32_t, std::string> pages_;
+};
+
+}  // namespace bftEngine::test

ccron/test/ccron_table_test.cpp

@@ -19,8 +19,8 @@
 #include "ccron/cron_table.hpp"
 #include "ccron/periodic_action.hpp"
 
-#include "IReservedPages.hpp"
 #include "ReservedPagesClient.hpp"
+#include "ReservedPagesMock.hpp"
 
 #include <algorithm>
 #include <cstdint>
@@ -34,46 +34,6 @@ namespace {
 using namespace concord::cron;
 using namespace std::chrono_literals;
 
-struct ReservedPagesMock : public bftEngine::IReservedPages {
-  uint32_t numberOfReservedPages() const override { return 1; }
-
-  uint32_t sizeOfReservedPage() const override { return 4096; }
-
-  bool loadReservedPage(uint32_t page_id, uint32_t size, char* data) const override {
-    auto it = pages_.find(page_id);
-    if (it != pages_.cend()) {
-      std::memcpy(data, it->second.data(), std::min<std::size_t>(size, it->second.size()));
-      return true;
-    }
-    return false;
-  }
-
-  void saveReservedPage(uint32_t page_id, uint32_t size, const char* data) override {
-    pages_[page_id].assign(data, std::min<std::size_t>(size, sizeOfReservedPage()));
-  }
-
-  void zeroReservedPage(uint32_t page_id) override {
-    auto it = pages_.find(page_id);
-    if (it != pages_.cend()) {
-      for (auto i = 0ul; i < it->second.size(); ++i) {
-        it->second[i] = '\0';
-      }
-    }
-  }
-
-  bool isReservedPageZeroed(uint32_t page_id) const {
-    auto it = pages_.find(cron_res_pages_client_.my_offset() + page_id);
-    if (it == pages_.cend()) {
-      return false;
-    }
-    return (it->second.find_first_not_of('\0') == std::string::npos);
-  }
-
-  // reserved page ID -> contents
-  std::map<uint32_t, std::string> pages_;
-  CronResPagesClient cron_res_pages_client_;
-};
-
 class ccron_table_test : public ::testing::Test {
  protected:
   const std::uint32_t kComponentId = 99;
@@ -85,7 +45,7 @@ class ccron_table_test : public ::testing::Test {
   std::vector<std::uint32_t> schedule_next_called_;
   std::vector<std::uint32_t> on_remove_called_;
 
-  ReservedPagesMock res_pages_mock_;
+  bftEngine::test::ReservedPagesMock<CronResPagesClient> res_pages_mock_;
 
   void SetUp() override { bftEngine::ReservedPagesClientBase::setReservedPages(&res_pages_mock_); }
 
@@ -133,8 +93,8 @@ class ccron_table_test : public ::testing::Test {
 
   PeriodicActionSchedule getScheduleFromResPage() const {
     auto schedule = PeriodicActionSchedule{};
-    EXPECT_EQ(1, res_pages_mock_.pages_.size());
-    const auto& page = res_pages_mock_.pages_.begin()->second;
+    EXPECT_EQ(1, res_pages_mock_.pages().size());
+    const auto& page = res_pages_mock_.pages().begin()->second;
     auto page_ptr = reinterpret_cast<const uint8_t*>(page.data());
     deserialize(page_ptr, page_ptr + page.size(), schedule);
     EXPECT_EQ(1, schedule.components.size());

kvbc/CMakeLists.txt

@@ -21,6 +21,7 @@ add_library(kvbc  src/ClientImp.cpp
     src/direct_kv_storage_factory.cpp
     src/merkle_tree_storage_factory.cpp
     src/pruning_handler.cpp
+    src/pruning_reserved_pages_client.cpp
     src/st_reconfiguration_sm.cpp
     src/sparse_merkle/base_types.cpp
     src/sparse_merkle/keys.cpp
@@ -41,7 +42,7 @@ if (BUILD_ROCKSDB_STORAGE)
 
 endif (BUILD_ROCKSDB_STORAGE)
 target_link_libraries(kvbc PUBLIC corebft util)
-target_link_libraries(kvbc PUBLIC categorized_kvbc_msgs)
+target_link_libraries(kvbc PUBLIC categorized_kvbc_msgs pruning_msgs)
 target_link_libraries(kvbc PUBLIC concord_block_update concord-kvbc-proto)
 
 target_include_directories(kvbc PUBLIC include util)

kvbc/cmf/CMakeLists.txt

@@ -2,3 +2,8 @@ cmf_generate_cpp(header cpp concord::kvbc::categorization categorized_kvbc_msgs.
 add_library(categorized_kvbc_msgs ${cpp})
 set_target_properties(categorized_kvbc_msgs PROPERTIES LINKER_LANGUAGE CXX)
 target_include_directories(categorized_kvbc_msgs PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+cmf_generate_cpp(header cpp concord::kvbc::pruning pruning_msgs.cmf)
+add_library(pruning_msgs ${cpp})
+set_target_properties(pruning_msgs PROPERTIES LINKER_LANGUAGE CXX)
+target_include_directories(pruning_msgs PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

kvbc/cmf/pruning_msgs.cmf

@@ -0,0 +1,17 @@
+# Represents a pruning agreement, i.e. when processing a PruneRequest.
+Msg Agreement 1 {
+  # TicksGenerator period in seconds.
+  uint32 tick_period_seconds
+
+  # The number of blocks in a pruning batch.
+  uint64 batch_blocks_num
+
+  # The last agreed prunable block ID for all replicas.
+  uint64 last_agreed_prunable_block_id
+}
+
+# Represents a pruning batch.
+Msg Batch 2 {
+  # The latest block ID to prune in the batch.
+  uint64 latest_batch_block_id_to
+}

kvbc/include/categorization/blockchain.h

@@ -21,7 +21,6 @@ namespace concord::kvbc::categorization::detail {
 class Blockchain {
  public:
   static constexpr auto MAX_BLOCK_ID = std::numeric_limits<BlockId>::max();
-  static constexpr auto INITIAL_GENESIS_BLOCK_ID = BlockId{1};
   static constexpr auto INVALID_BLOCK_ID = BlockId{0};
 
   Blockchain(const std::shared_ptr<concord::storage::rocksdb::NativeClient>& native_client);

kvbc/include/db_adapter_interface.h

@@ -20,8 +20,6 @@ class NotFoundException : public std::runtime_error {
   const char* what() const noexcept override { return std::runtime_error::what(); }
 };
 
-inline constexpr auto INITIAL_GENESIS_BLOCK_ID = BlockId{1};
-
 class IDbAdapter {
  public:
   // Returns the added block ID.

kvbc/include/kv_types.hpp

@@ -35,6 +35,8 @@ typedef std::set<Key> OrderedKeysSet;
 typedef KeysVector ValuesVector;
 typedef std::uint64_t BlockId;
 
+inline constexpr auto INITIAL_GENESIS_BLOCK_ID = BlockId{1};
+
 template <typename ContainerIn>
 OrderedSetOfKeyValuePairs order(const ContainerIn& unordered) {
   OrderedSetOfKeyValuePairs out;

kvbc/include/pruning_reserved_pages_client.hpp

@@ -0,0 +1,107 @@
+// Concord
+//
+// Copyright (c) 2021 VMware, Inc. All Rights Reserved.
+//
+// This product is licensed to you under the Apache 2.0 license (the "License").  You may not use this product except in
+// compliance with the Apache 2.0 License.
+//
+// This product may include a number of subcomponents with separate copyright notices and license terms. Your use of
+// these subcomponents is subject to the terms and conditions of the sub-component's license, as noted in the LICENSE
+// file.
+
+#pragma once
+
+#include "ReservedPagesClient.hpp"
+
+#include "kv_types.hpp"
+
+#include "pruning_msgs.cmf.hpp"
+
+#include <cstdint>
+#include <chrono>
+
+namespace concord::kvbc::pruning {
+
+// Creates a pruning agreement.
+// `tick_period` must be at least 1 second.
+// `batch_blocks_num` must be non-zero.
+// `last_agreed_prunable_block_id` must be at least INITIAL_GENESIS_BLOCK_ID.
+Agreement createAgreement(const std::chrono::seconds& tick_period,
+                          std::uint64_t batch_blocks_num,
+                          BlockId last_agreed_prunable_block_id);
+
+// Provides persistence of pruning agreement and progress information in reserved pages.
+//
+// Pseudo code for how it can be used:
+//
+// * On PruneRequest:
+//   client.saveAgreement(agreement);
+//   if (genesis <= agreement.last_agreed_prunable_block_id) ticks_gen.start();
+//
+// * On Tick:
+//   auto agreement = client.lastAgreement();
+//   if (agreement) {
+//     auto until = std::min(genesis + agreement->batch_blocks_num, agreement->last_agreed_prunable_block_id + 1);
+//     if (until >= genesis) {
+//       client.saveLatestBatch(until - 1);
+//       blockchain.deleteBlocksUntil(until);
+//     }
+//     if (genesis >= agreement->last_agreed_prunable_block_id) ticks_gen.stop();
+//   }
+//
+// * On Startup and State Transfer:
+//   auto agreement = client.lastAgreement();
+//   const auto to = client.latestBatchBlockIdTo();
+//   if (to && *to + 1 > genesis) blockchain.deleteUntil(*to + 1);
+//   if (genesis < agreement->last_agreed_prunable_block_id) ticks_gen.start();
+//   else ticks_gen.stop();
+class ReservedPagesClient {
+ private:
+  static constexpr auto kPruningResPageCount = 2;
+  using ClientType = bftEngine::ResPagesClient<ReservedPagesClient, kPruningResPageCount>;
+
+ public:
+  // Returns the number of reserved pages for this client.
+  static std::uint32_t numberOfReservedPagesForClient() { return ClientType::numberOfReservedPagesForClient(); }
+
+ public:
+  // Loads data from reserved pages on construction.
+  ReservedPagesClient();
+
+ public:
+  // Saves the given agreement to reserved pages.
+  void saveAgreement(const Agreement& agreement);
+
+  // Updates an existing agreement's `tick_period` and `batch_blocks_num` in reserved pages.
+  // Precondition: an agreement must already exist. If not, behaviour is undefined.
+  void updateExistingAgreement(const std::chrono::seconds& tick_period, std::uint64_t batch_blocks_num);
+
+  // Updates an existing agreement's `tick_period` in reserved pages.
+  // Precondition: an agreement must already exist. If not, behaviour is undefined.
+  void updateExistingAgreement(const std::chrono::seconds& tick_period);
+
+  // Updates an existing agreement's `batch_blocks_num` in reserved pages.
+  // Precondition: an agreement must already exist. If not, behaviour is undefined.
+  void updateExistingAgreement(std::uint64_t batch_blocks_num);
+
+  // Saves the `to` block ID of the latest batch. A batch is a range [genesis, to].
+  void saveLatestBatch(BlockId to);
+
+ public:
+  // Returns the latest agreement or std::nullopt if no agreement has been reached yet.
+  const std::optional<Agreement>& latestAgreement() const { return latest_agreement_; }
+
+  // Returns the `to` block ID of the latest batch or std::nullopt if no batch pruning has started yet.
+  const std::optional<BlockId>& latestBatchBlockIdTo() const { return latest_batch_block_id_to_; }
+
+ private:
+  static constexpr std::uint32_t kLatestAgreementPageId{0};
+  static constexpr std::uint32_t kLatestBatchBlockIdToPageId{1};
+
+ private:
+  std::optional<Agreement> latest_agreement_;
+  std::optional<BlockId> latest_batch_block_id_to_;
+  ClientType client_;
+};
+
+}  // namespace concord::kvbc::pruning