Skip to content

Commit

Permalink
Update Ethash cache implementation (#992)
Browse files Browse the repository at this point in the history 
* add CacheEntry serde implementation

* update progpow implementation

* Update static variables

* Replace new with make_unique

* fix make_unique

* add cache_entry test

* update

* fix comments

Co-authored-by: Bohdan Vanieiev <[email protected]>
  • Loading branch information
@Mr-Leshiy @Warchant
Mr-Leshiy and Warchant authored Aug 8, 2022
1 parent fd650b0 commit 604697b
Show file tree
Hide file tree
Showing 11 changed files with 114 additions and 98 deletions.
6 changes: 2 additions & 4 deletions include/veriblock/pop/algorithm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,7 @@ bool same_vectors_unique_unordered(const std::vector<T>& a,

//! @private
template <typename T>
bool same_vectors_unordered(const std::vector<T>& a,
const std::vector<T>& b) {

bool same_vectors_unordered(const std::vector<T>& a, const std::vector<T>& b) {
std::vector<T> sortedVectorA = a;
std::vector<T> sortedVectorB = b;
std::sort(sortedVectorA.begin(), sortedVectorA.end());
Expand Down Expand Up @@ -132,7 +130,7 @@ void erase_if(container_t& c, std::function<bool(const val_t&)> pred) {
//! @private
template <typename T, typename... Args>
std::unique_ptr<T> make_unique(Args&&... args) {
auto* ptr = new T(std::forward<Args...>(args)...);
auto* ptr = new T(std::forward<Args>(args)...);
return std::unique_ptr<T>(ptr);
}

Expand Down
11 changes: 5 additions & 6 deletions include/veriblock/pop/crypto/progpow.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,9 @@
#ifndef VERIBLOCK_POP_CPP_PROGPOW_HPP
#define VERIBLOCK_POP_CPP_PROGPOW_HPP

#include <veriblock/pop/serde.hpp>
#include <veriblock/pop/storage/ethash_cache_provider.hpp>
#include <veriblock/pop/storage/progpow_header_cache_provider.hpp>
#include <veriblock/pop/uint.hpp>
#include "progpow/cache.hpp"
#include "veriblock/pop/serde.hpp"
#include "veriblock/pop/uint.hpp"

namespace altintegration {

Expand Down Expand Up @@ -75,9 +74,9 @@ void clearEthashCache();

} // namespace progpow

void setEthashCache(const std::shared_ptr<EthashCache>& cache);
void setEthashCache(std::unique_ptr<EthashCacheI> cache);

void setProgpowHeaderCache(const std::shared_ptr<ProgpowHeaderCache>& cache);
void setProgpowHeaderCache(std::unique_ptr<ProgpowHeaderCacheI> cache);

/**
* Calculate vPROGPOW hash of given VbkBlock header (65 bytes)
Expand Down
4 changes: 2 additions & 2 deletions include/veriblock/pop/pop_context.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ struct PopContext {
std::shared_ptr<Config> config,
std::shared_ptr<PayloadsStorage> payloadsProvider,
std::shared_ptr<BlockReader> blockProvider,
const std::shared_ptr<EthashCache>& ethashCache,
const std::shared_ptr<ProgpowHeaderCache>& progpowHeaderCache,
std::unique_ptr<EthashCache> ethashCache,
std::unique_ptr<ProgpowHeaderCache> progpowHeaderCache,
size_t validatorWorkers = 0);

/**
Expand Down
14 changes: 7 additions & 7 deletions include/veriblock/pop/serde.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ struct AltChainParams;
* @param state will return error description here
* @return true if check is OK, false otherwise
*/
bool checkRange(int64_t num, int64_t min, int64_t max, ValidationState& state);
bool checkRange(uint64_t num, uint64_t min, uint64_t max, ValidationState& state);

/**
* Converts the input to the byte array and trims it's size to the
Expand Down Expand Up @@ -72,8 +72,8 @@ std::vector<uint8_t> fixedArray(T input) {
bool readVarLenValue(ReadStream& stream,
Slice<const uint8_t>& out,
ValidationState& state,
size_t minLen,
size_t maxLen);
uint64_t minLen,
uint64_t maxLen);

/**
* Read variable length value, which consists of
Expand All @@ -89,8 +89,8 @@ bool readVarLenValue(ReadStream& stream,
bool readSingleByteLenValue(ReadStream& stream,
Slice<const uint8_t>& out,
ValidationState& state,
size_t minLen,
size_t maxLen);
uint64_t minLen,
uint64_t maxLen);

//! @overload
template <typename Container,
Expand All @@ -99,8 +99,8 @@ template <typename Container,
bool readSingleByteLenValue(ReadStream& stream,
Container& out,
ValidationState& state,
size_t minLen,
size_t maxLen) {
uint64_t minLen,
uint64_t maxLen) {
uint8_t length = 0;
if (!stream.readBE<uint8_t>(length, state)) {
return state.Invalid("readsingle-bad-length");
Expand Down
18 changes: 14 additions & 4 deletions src/pop/crypto/progpow/cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@
namespace altintegration {

void CacheEntry::toVbkEncoding(WriteStream& stream) const {
// TODO implement
assert(false && "unimplemented");
stream.writeBE<uint64_t>(this->light->cache_size);
stream.writeBE<uint64_t>(this->light->epoch);
stream.write(this->light->cache, this->light->cache_size);

writeContainer(
stream, this->dag, [](WriteStream& stream, const uint32_t& value) {
stream.writeBE(value);
Expand All @@ -30,8 +32,16 @@ void CacheEntry::toVbkEncoding(WriteStream& stream) const {
bool DeserializeFromVbkEncoding(ReadStream& stream,
CacheEntry& out,
ValidationState& state) {
// TODO implement
assert(false && "unimplemented");
if (!stream.readBE<uint64_t>(out.light->cache_size, state)) {
return state.Invalid("invalid-cache_size");
}
if (!stream.readBE<uint64_t>(out.light->epoch, state)) {
return state.Invalid("invalid-epoch");
}
if (!stream.read(out.light->cache_size, (uint8_t*)out.light->cache, state)) {
return state.Invalid("invalid-cache");
}

size_t i = 0;
if (!readArrayOf<uint32_t>(
stream,
Expand Down
94 changes: 36 additions & 58 deletions src/pop/crypto/progpow/progpow.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@
// https://www.veriblock.org
// Distributed under the MIT software license, see the accompanying
// file LICENSE or http://www.opensource.org/licenses/mit-license.php.
#include <algorithm>
#include <array>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <limits>
#include <memory>
#include <mutex>
#include <string>
#include <utility>
#include <vector>
#include <veriblock/pop/assert.hpp>
#include <veriblock/pop/cache/small_lfru_cache.hpp>
#include <veriblock/pop/consts.hpp>
Expand All @@ -14,21 +26,9 @@
#include <veriblock/pop/slice.hpp>
#include <veriblock/pop/third_party/lru_cache.hpp>
#include <veriblock/pop/trace.hpp>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <mutex>
#include <utility>
#include <vector>
#include <algorithm>
#include <array>
#include <functional>
#include <limits>
#include <memory>
#include <string>

#include "libethash/internal.hpp"

#include "veriblock/pop/algorithm.hpp"
#include "veriblock/pop/blob.hpp"
#include "veriblock/pop/crypto/endian.hpp"
#include "veriblock/pop/crypto/vblake.hpp"
Expand Down Expand Up @@ -615,104 +615,80 @@ std::string hash32_t::toHex() const {
using LockGuard = std::lock_guard<VBK_TRACE_LOCKABLE_BASE(std::mutex)>;

// epoch -> ethash cache + dag
struct EthashCache_t : public EthashCacheI {
struct EthashCacheInMem : public EthashCacheI {
std::shared_ptr<CacheEntry> getOrDefault(
uint64_t epoch,
std::function<std::shared_ptr<CacheEntry>()> factory) override {
return this->in_memory_cache.getOrDefault(epoch, factory);
}

void clear() override {
this->in_memory_cache.clear();
if (this->on_disk_cache != nullptr) {
this->on_disk_cache->clear();
}
}

void setOnDiskCache(const std::shared_ptr<EthashCache>& cache) {
this->on_disk_cache = cache;
}
void clear() override { return this->in_memory_cache.clear(); }

private:
cache::SmallLFRUCache<uint64_t, CacheEntry, VBK_PROGPOW_ETHASH_CACHE_SIZE>
in_memory_cache{};

std::shared_ptr<EthashCacheI> on_disk_cache{nullptr};
};

static EthashCache_t ethash_cache{};
static std::unique_ptr<EthashCacheI> ethash_cache =
make_unique<EthashCacheInMem>();

// protects EthashCache
static VBK_TRACE_LOCKABLE_BASE(std::mutex) & GetEthashCacheMutex() {
static VBK_TRACE_LOCKABLE(std::mutex, csEthashCache);
return csEthashCache;
}

void setEthashCache(const std::shared_ptr<EthashCache>& cache) {
void setEthashCache(std::unique_ptr<EthashCacheI> cache) {
LockGuard lock(GetEthashCacheMutex());
ethash_cache.setOnDiskCache(cache);
if (cache != nullptr) {
ethash_cache = std::move(cache);
}
}

static EthashCacheI& GetEthashCache() {
// NOLINTNEXTLINE(cert-err58-cpp)
return ethash_cache;
VBK_ASSERT(ethash_cache != nullptr);
return *ethash_cache;
}

// sha256d(vbkheader) -> progpow hash
struct ProgpowHeaderCache_T : public ProgpowHeaderCacheI {
ProgpowHeaderCache_T(size_t maxSize, size_t elasticity)
: in_memory_cache(maxSize, elasticity), on_disk_cache{nullptr} {}
struct ProgpowHeaderCacheInMem : public ProgpowHeaderCacheI {
ProgpowHeaderCacheInMem(size_t maxSize, size_t elasticity)
: in_memory_cache(maxSize, elasticity) {}

void insert(const uint256& key, uint192 value) override {
if (this->on_disk_cache != nullptr) {
return this->on_disk_cache->insert(key, value);
}

return this->in_memory_cache.insert(key, value);
}

bool tryGet(const uint256& key, uint192& value) override {
if (this->on_disk_cache != nullptr) {
return this->on_disk_cache->tryGet(key, value);
}

return this->in_memory_cache.tryGet(key, value);
}

void clear() override {
if (this->on_disk_cache != nullptr) {
return this->on_disk_cache->clear();
}

return this->in_memory_cache.clear();
}

void setOnDiskCache(const std::shared_ptr<ProgpowHeaderCache>& cache) {
this->on_disk_cache = cache;
}
void clear() override { return this->in_memory_cache.clear(); }

private:
lru11::Cache<uint256, uint192, std::mutex> in_memory_cache;
std::shared_ptr<ProgpowHeaderCacheI> on_disk_cache{nullptr};
};

static ProgpowHeaderCache_T progpow_header_cache(VBK_PROGPOW_HEADER_HASH_SIZE,
1000);
static std::unique_ptr<ProgpowHeaderCacheI> progpow_header_cache =
make_unique<ProgpowHeaderCacheInMem>(VBK_PROGPOW_HEADER_HASH_SIZE, 1000);

// protects ProgpowHeaderCache
static VBK_TRACE_LOCKABLE_BASE(std::mutex) & GetProgpowHeaderCacheMutex() {
static VBK_TRACE_LOCKABLE(std::mutex, csProgpowHeaderCache);
return csProgpowHeaderCache;
}

void setProgpowHeaderCache(const std::shared_ptr<ProgpowHeaderCache>& cache) {
void setProgpowHeaderCache(std::unique_ptr<ProgpowHeaderCacheI> cache) {
LockGuard lock(GetProgpowHeaderCacheMutex());
progpow_header_cache.setOnDiskCache(cache);
if (cache != nullptr) {
progpow_header_cache = std::move(cache);
}
}

// NOLINTNEXTLINE(cert-err58-cpp)
static ProgpowHeaderCacheI& GetProgpowHeaderCache() {
return progpow_header_cache;
return *progpow_header_cache;
}

void progpow::insertHeaderCacheEntry(Slice<const uint8_t> header,
Expand All @@ -727,10 +703,12 @@ void progpow::clearHeaderCache() {
LockGuard lock(GetProgpowHeaderCacheMutex());
GetProgpowHeaderCache().clear();
}

void progpow::clearEthashCache() {
LockGuard lock(GetEthashCacheMutex());
GetEthashCache().clear();
}

static uint192 progPowHashImpl(Slice<const uint8_t> header) {
VBK_ASSERT(header.size() == VBK_HEADER_SIZE_PROGPOW);
const auto height = progpow::getVbkBlockHeight(header);
Expand Down
8 changes: 4 additions & 4 deletions src/pop/pop_context.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,11 @@ std::shared_ptr<PopContext> PopContext::create(
std::shared_ptr<Config> config,
std::shared_ptr<PayloadsStorage> payloadsProvider,
std::shared_ptr<BlockReader> blockProvider,
const std::shared_ptr<EthashCache>& ethashCache,
const std::shared_ptr<ProgpowHeaderCache>& progpowHeaderCache,
std::unique_ptr<EthashCache> ethashCache,
std::unique_ptr<ProgpowHeaderCache> progpowHeaderCache,
size_t validatorWorkers) {
setEthashCache(ethashCache);
setProgpowHeaderCache(progpowHeaderCache);
setEthashCache(std::move(ethashCache));
setProgpowHeaderCache(std::move(progpowHeaderCache));

config->validate();

Expand Down
16 changes: 8 additions & 8 deletions src/pop/serde.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

namespace altintegration {

bool checkRange(int64_t num, int64_t min, int64_t max, ValidationState& state) {
bool checkRange(uint64_t num, uint64_t min, uint64_t max, ValidationState& state) {
if (num < min) {
return state.Invalid(
"range-below",
Expand Down Expand Up @@ -58,8 +58,8 @@ std::vector<uint8_t> trimmedArray(int64_t input) {
bool readVarLenValue(ReadStream& stream,
Slice<const uint8_t>& out,
ValidationState& state,
size_t minLen,
size_t maxLen) {
uint64_t minLen,
uint64_t maxLen) {
int32_t length = 0;
if (!readSingleBEValue<int32_t>(stream, length, state)) {
return state.Invalid("readvarlen-bad-length");
Expand All @@ -73,8 +73,8 @@ bool readVarLenValue(ReadStream& stream,
bool readSingleByteLenValue(ReadStream& stream,
Slice<const uint8_t>& out,
ValidationState& state,
size_t minLen,
size_t maxLen) {
uint64_t minLen,
uint64_t maxLen) {
uint8_t length = 0;
if (!stream.readBE<uint8_t>(length, state)) {
return state.Invalid("readsingle-bad-length");
Expand All @@ -88,7 +88,7 @@ bool readSingleByteLenValue(ReadStream& stream,
void writeSingleByteLenValue(WriteStream& stream, Slice<const uint8_t> value) {
ValidationState state;
VBK_ASSERT_MSG(
checkRange(value.size(), 0, (std::numeric_limits<uint8_t>::max)(), state),
checkRange(value.size(), 0, (uint64_t)(std::numeric_limits<uint8_t>::max)(), state),
"Can not writeSingleByteLen: " + state.toString());
stream.writeBE<uint8_t>((uint8_t)value.size());
stream.write(value);
Expand All @@ -108,7 +108,7 @@ void writeVarLenValue(WriteStream& stream, Slice<const uint8_t> value) {
size_t singleByteLenValueSize(Slice<const uint8_t> value) {
ValidationState state;
VBK_ASSERT_MSG(
checkRange(value.size(), 0, (std::numeric_limits<uint8_t>::max)(), state),
checkRange(value.size(), 0, (uint64_t)(std::numeric_limits<uint8_t>::max)(), state),
"Can not singleByteLenSize: " + state.toString());
size_t size = 0;
size += sizeof((uint8_t)value.size());
Expand All @@ -119,7 +119,7 @@ size_t singleByteLenValueSize(Slice<const uint8_t> value) {
size_t singleByteLenValueSize(size_t valueSize) {
ValidationState state;
VBK_ASSERT_MSG(
checkRange(valueSize, 0, (std::numeric_limits<uint8_t>::max)(), state),
checkRange(valueSize, 0, (uint64_t)(std::numeric_limits<uint8_t>::max)(), state),
"Can not singleByteLenSize: " + state.toString());
size_t size = 0;
size += sizeof((uint8_t)valueSize);
Expand Down
1 change: 1 addition & 0 deletions test/pop/crypto/progpow/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ addtest(progpow_test
progpow_test.cpp
${progpow_test_vectors}
)
addtest(cache_entry_test cache_entry_test.cpp)
set_test_cost(progpow_test 38000) # 38 sec
addtest(ethash_dag_test ethash_dag_test.cpp)
set_test_cost(ethash_dag_test 21000) # 21 sec
Expand Down
Loading

0 comments on commit 604697b

Please sign in to comment.