From cfa084f42b3f5d4020eff0eb2b9550f04a960489 Mon Sep 17 00:00:00 2001 From: wanghenshui Date: Sat, 18 Feb 2023 20:40:39 +0800 Subject: [PATCH] feat: add rate limiter config --- conf/pika.conf | 3 +++ include/pika_conf.h | 2 ++ src/blackwidow/include/blackwidow/blackwidow.h | 16 +++++----------- src/pika_admin.cc | 6 ++++++ src/pika_conf.cc | 6 ++++++ src/pika_server.cc | 3 +++ 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/conf/pika.conf b/conf/pika.conf index 502ec2f831..0afec8d366 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -157,3 +157,6 @@ max-bytes-for-level-multiplier : 10 # optimize-filters-for-hits: no # https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size # level-compaction-dynamic-level-bytes: no + +# rate limiter bandwidth, default 200MB +#rate-limiter-bandwidth : 209715200 diff --git a/include/pika_conf.h b/include/pika_conf.h index ec4dcf8ea0..75e0c300c3 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -93,6 +93,7 @@ class PikaConf : public slash::BaseConf { int max_conn_rbuf_size() { return max_conn_rbuf_size_.load(); } int consensus_level() { return consensus_level_.load(); } int replication_num() { return replication_num_.load(); } + int64_t rate_limiter_bandwidth() { RWLock l(&rwlock_, false); return rate_limiter_bandwidth_; } // Immutable config items, we don't use lock. bool daemonize() { return daemonize_; } @@ -349,6 +350,7 @@ class PikaConf : public slash::BaseConf { bool pin_l0_filter_and_index_blocks_in_cache_; bool optimize_filters_for_hits_; bool level_compaction_dynamic_level_bytes_; + int64_t rate_limiter_bandwidth_ = 200 * 1024 * 1024; // 200M std::atomic sync_window_size_; std::atomic max_conn_rbuf_size_; std::atomic consensus_level_; diff --git a/src/blackwidow/include/blackwidow/blackwidow.h b/src/blackwidow/include/blackwidow/blackwidow.h index 905d1b9e1d..1d152a93d1 100644 --- a/src/blackwidow/include/blackwidow/blackwidow.h +++ b/src/blackwidow/include/blackwidow/blackwidow.h @@ -15,6 +15,7 @@ #include "rocksdb/status.h" #include "rocksdb/options.h" +#include "rocksdb/rate_limiter.h" #include "rocksdb/slice.h" #include "rocksdb/table.h" #include "rocksdb/filter_policy.h" @@ -62,17 +63,10 @@ class LRUCache; struct BlackwidowOptions { rocksdb::Options options; rocksdb::BlockBasedTableOptions table_options; - size_t block_cache_size; - bool share_block_cache; - size_t statistics_max_size; - size_t small_compaction_threshold; - - explicit BlackwidowOptions() - : block_cache_size(0), - share_block_cache(false), - statistics_max_size(0), - small_compaction_threshold(5000) {} - + size_t block_cache_size = 0; + bool share_block_cache = false; + size_t statistics_max_size = 0; + size_t small_compaction_threshold = 5000; Status ResetOptions(const OptionType& option_type, const std::unordered_map& options_map); }; diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 0b97ede04f..fdfcb2c031 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -1739,6 +1739,12 @@ void ConfigCmd::ConfigGet(std::string& ret) { EncodeInt32(&config_body, g_pika_conf->consensus_level()); } + if (slash::stringmatch(pattern.data(), "rate-limiter-bandwidth", 1)) { + elements += 2; + EncodeString(&config_body, "rate-limiter-bandwidth"); + EncodeInt64(&config_body, g_pika_conf->rate_limiter_bandwidth()); + } + std::stringstream resp; resp << "*" << std::to_string(elements) << "\r\n" << config_body; ret = resp.str(); diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 16ad536a15..b9b5a9f5cf 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -375,6 +375,12 @@ int PikaConf::Load() max_write_buffer_size_ = 10737418240; // 10Gb } + // rate-limiter-bandwidth + GetConfInt64("rate-limiter-bandwidth", &rate_limiter_bandwidth_); + if (rate_limiter_bandwidth_ <= 0 ) { + rate_limiter_bandwidth_ = 200 * 1024 * 1024; // 200MB + } + // max_write_buffer_num max_write_buffer_num_ = 2; GetConfInt("max-write-buffer-num", &max_write_buffer_num_); diff --git a/src/pika_server.cc b/src/pika_server.cc index 6b14df3474..ae4da3ace2 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1660,6 +1660,9 @@ void PikaServer::InitBlackwidowOptions() { rocksdb::NewLRUCache(bw_options_.block_cache_size, g_pika_conf->num_shard_bits()); } + bw_options_.options.rate_limiter = + std::shared_ptr(rocksdb::NewGenericRateLimiter(g_pika_conf->rate_limiter_bandwidth())); + // For Blackwidow small compaction bw_options_.statistics_max_size = g_pika_conf->max_cache_statistic_keys(); bw_options_.small_compaction_threshold =