Skip to content

Commit

Permalink
Update configuratio to handle a global write_buffer_manager, improve …
Browse files Browse the repository at this point in the history 
…error handling around configuration
  • Loading branch information
@sgerbino
sgerbino committed May 1, 2019
1 parent 4dd53f7 commit 0f8907e
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 43 deletions.
87 changes: 77 additions & 10 deletions libraries/mira/src/configuration.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
#include <mira/configuration.hpp>

#define DEFAULT_MIRA_SHARED_CACHE_SIZE (1ull * 1024 * 1024 * 1024 )
#define DEFAULT_MIRA_NUM_SHARD_BITS (4)

namespace mira {

// Base configuration for an index
Expand All @@ -11,9 +8,13 @@ namespace mira {
// Global options
#define GLOBAL "global"
#define SHARED_CACHE "shared_cache"
#define WRITE_BUFFER_MANAGER "write_buffer_manager"
#define OBJECT_COUNT "object_count"
#define STATISTICS "statistics"

// Write buffer manager options
#define WRITE_BUFFER_SIZE "write_buffer_size"

// Shared cache options
#define CAPACITY "capacity"
#define NUM_SHARD_BITS "num_shard_bits"
Expand All @@ -34,8 +35,10 @@ namespace mira {
#define BLOOM_FILTER_POLICY "bloom_filter_policy"
#define BITS_PER_KEY "bits_per_key"
#define USE_BLOCK_BASED_BUILDER "use_block_based_builder"
#define CACHE_INDEX_AND_FILTER_BLOCKS "cache_index_and_filter_blocks"

static std::shared_ptr< rocksdb::Cache > global_shared_cache;
static std::shared_ptr< rocksdb::WriteBufferManager > global_write_buffer_manager;

static std::map< std::string, std::function< void( ::rocksdb::Options&, fc::variant ) > > global_database_option_map {
{ ALLOW_MMAP_READS, []( ::rocksdb::Options& o, fc::variant v ) { o.allow_mmap_reads = v.as< bool >(); } },
Expand Down Expand Up @@ -69,7 +72,20 @@ static std::map< std::string, std::function< void( ::rocksdb::Options&, fc::vari
table_options.block_cache = global_shared_cache;

if ( obj.contains( BLOCK_SIZE ) )
{
FC_ASSERT( obj[ BLOCK_SIZE ].is_uint64(), "Expected '${key}' to be an unsigned integer",
("key", BLOCK_SIZE) );

table_options.block_size = obj[ BLOCK_SIZE ].template as< uint64_t >();
}

if ( obj.contains( CACHE_INDEX_AND_FILTER_BLOCKS ) )
{
FC_ASSERT( obj[ CACHE_INDEX_AND_FILTER_BLOCKS ].is_bool(), "Expected '${key}' to be a boolean",
("key", CACHE_INDEX_AND_FILTER_BLOCKS) );

table_options.cache_index_and_filter_blocks = obj[ CACHE_INDEX_AND_FILTER_BLOCKS ].template as< bool >();
}

if ( obj.contains( BLOOM_FILTER_POLICY ) )
{
Expand All @@ -84,12 +100,22 @@ static std::map< std::string, std::function< void( ::rocksdb::Options&, fc::vari
("parent", BLOOM_FILTER_POLICY)
("key", BITS_PER_KEY) );

FC_ASSERT( filter_policy[ BITS_PER_KEY ].is_uint64(), "Expected '${key}' to be an unsigned integer",
("key", BITS_PER_KEY) );

bits_per_key = filter_policy[ BITS_PER_KEY ].template as< uint64_t >();

if ( filter_policy.contains( USE_BLOCK_BASED_BUILDER ) )
{
FC_ASSERT( filter_policy[ USE_BLOCK_BASED_BUILDER ].is_bool(), "Expected '${key}' to be a boolean",
("key", USE_BLOCK_BASED_BUILDER) );

table_options.filter_policy.reset( rocksdb::NewBloomFilterPolicy( bits_per_key, filter_policy[ USE_BLOCK_BASED_BUILDER ].template as< bool >() ) );
}
else
{
table_options.filter_policy.reset( rocksdb::NewBloomFilterPolicy( bits_per_key ) );
}
}

o.table_factory.reset( ::rocksdb::NewBlockBasedTableFactory( table_options ) );
Expand Down Expand Up @@ -214,19 +240,60 @@ ::rocksdb::Options configuration::get_options( const boost::any& cfg, std::strin

auto& shared_cache_obj = global_config[ SHARED_CACHE ].get_object();

if ( shared_cache_obj.contains( CAPACITY ) && shared_cache_obj[ CAPACITY ].is_uint64() )
capacity = shared_cache_obj[ CAPACITY ].as< uint64_t >();
else
capacity = DEFAULT_MIRA_SHARED_CACHE_SIZE;
FC_ASSERT( shared_cache_obj.contains( CAPACITY ), "Expected '${parent}' configuration to contain '${key}'",
("parent", SHARED_CACHE)
("key", CAPACITY) );

FC_ASSERT( shared_cache_obj[ CAPACITY ].is_string(), "Expected '${key}' to be a string representation of an unsigned integer",
("key", CAPACITY) );

capacity = shared_cache_obj[ CAPACITY ].as< uint64_t >();

if ( shared_cache_obj.contains( NUM_SHARD_BITS ) )
{
FC_ASSERT( shared_cache_obj[ NUM_SHARD_BITS ].is_uint64(), "Expected '${key}' to be an unsigned integer",
("key", NUM_SHARD_BITS) );

if ( shared_cache_obj.contains( NUM_SHARD_BITS ) && shared_cache_obj[ NUM_SHARD_BITS ].is_uint64() )
num_shard_bits = shared_cache_obj[ NUM_SHARD_BITS ].as< int >();

global_shared_cache = rocksdb::NewLRUCache( capacity, num_shard_bits );
}
else
num_shard_bits = DEFAULT_MIRA_NUM_SHARD_BITS;
{
global_shared_cache = rocksdb::NewLRUCache( capacity );
}
}

if ( global_write_buffer_manager == nullptr )
{
size_t write_buf_size = 0;

fc::variant_object global_config = retrieve_global_configuration( obj );

global_shared_cache = rocksdb::NewLRUCache( capacity, num_shard_bits );
FC_ASSERT( global_config.contains( WRITE_BUFFER_MANAGER ), "Expected '${parent}' configuration to contain '${key}'",
("parent", GLOBAL)
("key", WRITE_BUFFER_MANAGER) );

FC_ASSERT( global_config[ WRITE_BUFFER_MANAGER ].is_object(), "Expected '${key}' to be an object",
("key", WRITE_BUFFER_MANAGER) );

auto& write_buffer_mgr_obj = global_config[ WRITE_BUFFER_MANAGER ].get_object();

FC_ASSERT( write_buffer_mgr_obj.contains( WRITE_BUFFER_SIZE ), "Expected '${parent}' configuration to contain '${key}'",
("parent", WRITE_BUFFER_MANAGER)
("key", WRITE_BUFFER_SIZE) );

FC_ASSERT( write_buffer_mgr_obj[ WRITE_BUFFER_SIZE ].is_string(), "Expected '${key}' to be a string representation of an unsigned integer",
("key", WRITE_BUFFER_SIZE) );

write_buf_size = write_buffer_mgr_obj[ WRITE_BUFFER_SIZE ].as< uint64_t >();

global_write_buffer_manager = std::make_shared< ::rocksdb::WriteBufferManager >( write_buf_size, global_shared_cache );
}

// We assign the global write buffer manager to all databases
opts.write_buffer_manager = global_write_buffer_manager;

fc::variant_object config = retrieve_active_configuration( obj, type_name );

for ( auto it = config.begin(); it != config.end(); ++it )
Expand Down
4 changes: 2 additions & 2 deletions libraries/plugins/chain/chain_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,9 +476,9 @@ void chain_plugin::plugin_startup()
{
database_config = fc::json::from_file( my->database_cfg, fc::json::strict_parser );
}
catch ( const std::exception& e )
catch ( ... )
{
elog( "Error while parsing database configuration: ${e}", ("e", e.what()) );
elog( "Error while parsing database configuration" );
exit( EXIT_FAILURE );
}
#endif
Expand Down
58 changes: 27 additions & 31 deletions libraries/utilities/database_configuration.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,25 @@
#include <fc/reflect/variant.hpp>
#include <steem/utilities/database_configuration.hpp>

#define KB(x) ((size_t) (x) << 10)
#define MB(x) ((size_t) (x) << 20)
#define GB(x) ((size_t) (x) << 30)

namespace steem { namespace utilities {

namespace database { namespace configuration {

struct shared_cache {
uint64_t capacity;
int num_shared_bits;
std::string capacity;
};

struct write_buffer_manager {
std::string write_buffer_size;
};

struct global {
database::configuration::shared_cache shared_cache;
database::configuration::write_buffer_manager write_buffer_manager;
uint64_t object_count;
bool statistics;
};
Expand All @@ -25,17 +33,11 @@ struct bloom_filter_policy {

struct block_based_table_options {
uint64_t block_size;
bool cache_index_and_filter_blocks;
database::configuration::bloom_filter_policy bloom_filter_policy;
};

struct base_index {
bool allow_mmap_reads;
uint64_t write_buffer_size;
uint64_t max_bytes_for_level_base;
uint64_t target_file_size_base;
int max_write_buffer_number;
int max_background_compactions;
int max_background_flushes;
bool optimize_level_style_compaction;
bool increase_parallelism;
database::configuration::block_based_table_options block_based_table_options;
Expand All @@ -46,36 +48,32 @@ struct configuration {
database::configuration::base_index base;
};

} } // indices::configuration
} } // database::configuration

fc::variant default_database_configuration()
{
database::configuration::configuration config;

// global
config.global.object_count = 40000000;
config.global.statistics = false;
config.global.object_count = 62500; // 4GB heaviest usage
config.global.statistics = false; // Incurs severe performance degradation when true

// global::shared_cache
config.global.shared_cache.capacity = 1073741824;
config.global.shared_cache.num_shared_bits = 4;
config.global.shared_cache.capacity = std::to_string( GB(1) );

// global::write_buffer_manager
config.global.write_buffer_manager.write_buffer_size = std::to_string( GB(1) ); // Write buffer manager is within the shared cache

// base
config.base.allow_mmap_reads = true;
config.base.write_buffer_size = 2097152;
config.base.max_bytes_for_level_base = 5242880;
config.base.target_file_size_base = 102400;
config.base.max_write_buffer_number = 16;
config.base.max_background_compactions = 16;
config.base.max_background_flushes = 16;
config.base.optimize_level_style_compaction = true;
config.base.increase_parallelism = true;

// base::block_based_table_options
config.base.block_based_table_options.block_size = 8192;
config.base.block_based_table_options.block_size = KB(8);
config.base.block_based_table_options.cache_index_and_filter_blocks = true;

// base::block_based_table_options::bloom_filter_policy
config.base.block_based_table_options.bloom_filter_policy.bits_per_key = 14;
config.base.block_based_table_options.bloom_filter_policy.bits_per_key = 10;
config.base.block_based_table_options.bloom_filter_policy.use_block_based_builder = false;

fc::variant config_obj;
Expand All @@ -88,11 +86,15 @@ fc::variant default_database_configuration()

FC_REFLECT( steem::utilities::database::configuration::shared_cache,
(capacity)
(num_shared_bits)
);

FC_REFLECT( steem::utilities::database::configuration::write_buffer_manager,
(write_buffer_size)
);

FC_REFLECT( steem::utilities::database::configuration::global,
(shared_cache)
(write_buffer_manager)
(object_count)
(statistics)
);
Expand All @@ -104,17 +106,11 @@ FC_REFLECT( steem::utilities::database::configuration::bloom_filter_policy,

FC_REFLECT( steem::utilities::database::configuration::block_based_table_options,
(block_size)
(cache_index_and_filter_blocks)
(bloom_filter_policy)
);

FC_REFLECT( steem::utilities::database::configuration::base_index,
(allow_mmap_reads)
(write_buffer_size)
(max_bytes_for_level_base)
(target_file_size_base)
(max_write_buffer_number)
(max_background_compactions)
(max_background_flushes)
(optimize_level_style_compaction)
(increase_parallelism)
(block_based_table_options)
Expand Down

0 comments on commit 0f8907e

Please sign in to comment.