automatically triggers a small compaction according statistics (OpenA…

…tomFoundation#395)

conf/pika.conf

@@ -93,6 +93,14 @@ binlog-file-size : 104857600
 # if this opsion is set to 'old', that means I will be a slave to Pika who's version 2.3.3 ~ 2.3.6
 # identify-binlog-type  [new | old]
 identify-binlog-type : new
+# Automatically triggers a small compaction according statistics
+# Use the cache to store up to 'max-cache-statistic-keys' keys
+# if 'max-cache-statistic-keys' set to '0', that means turn off the statistics function
+# it also doesn't automatically trigger a small compact feature
+max-cache-statistic-keys : 0
+# When 'delete' or 'overwrite' a specific multi-data structure key 'small-compaction-threshold' times,
+# a small compact is triggered automatically, default is 5000, limited in [1, 100000]
+small-compaction-threshold : 5000
 # Compression
 compression : snappy
 # max-background-flushes: default is 1, limited in [1, 4]

include/pika_conf.h

@@ -62,6 +62,8 @@ class PikaConf : public slash::BaseConf {
   }
   std::string compression()     { RWLock l(&rwlock_, false); return compression_; }
   int target_file_size_base()   { RWLock l(&rwlock_, false); return target_file_size_base_; }
+  int max_cache_statistic_keys() {RWLock l(&rwlock_, false); return max_cache_statistic_keys_;}
+  int small_compaction_threshold() {RWLock l(&rwlock_, false); return small_compaction_threshold_;}
   int max_background_flushes()  { RWLock l(&rwlock_, false); return max_background_flushes_; }
   int max_background_compactions()   { RWLock l(&rwlock_, false); return max_background_compactions_; }
   int max_cache_files()          { RWLock l(&rwlock_, false); return max_cache_files_; }
@@ -234,6 +236,8 @@ class PikaConf : public slash::BaseConf {
   int expire_logs_nums_;
   bool readonly_;
   std::string conf_path_;
+  int max_cache_statistic_keys_;
+  int small_compaction_threshold_;
   int max_background_flushes_;
   int max_background_compactions_;
   int max_cache_files_;

src/pika_admin.cc

@@ -1090,6 +1090,14 @@ void ConfigCmd::ConfigGet(std::string &ret) {
     ret = "*2\r\n";
     EncodeString(&ret, "target-file-size-base");
     EncodeInt32(&ret, g_pika_conf->target_file_size_base());
+  } else if (get_item == "max-cache-statistic-keys") {
+    ret = "*2\r\n";
+    EncodeString(&ret, "max-cache-statistic-keys");
+    EncodeInt32(&ret, g_pika_conf->max_cache_statistic_keys());
+  } else if (get_item == "small-compaction-threshold") {
+    ret = "*2\r\n";
+    EncodeString(&ret, "small-compaction-threshold");
+    EncodeInt32(&ret, g_pika_conf->small_compaction_threshold());
   } else if (get_item == "max-background-flushes") {
     ret = "*2\r\n";
     EncodeString(&ret, "max-background-flushes");
@@ -1187,7 +1195,7 @@ void ConfigCmd::ConfigGet(std::string &ret) {
     EncodeString(&ret, "slave-priority");
     EncodeInt32(&ret, g_pika_conf->slave_priority());
   } else if (get_item == "*") {
-    ret = "*104\r\n";
+    ret = "*108\r\n";
     EncodeString(&ret, "port");
     EncodeInt32(&ret, g_pika_conf->port());
     EncodeString(&ret, "double-master-ip");
@@ -1236,6 +1244,10 @@ void ConfigCmd::ConfigGet(std::string &ret) {
     EncodeInt32(&ret, g_pika_conf->maxclients());
     EncodeString(&ret, "target-file-size-base");
     EncodeInt32(&ret, g_pika_conf->target_file_size_base());
+    EncodeString(&ret, "max-cache-statistic-keys");
+    EncodeInt32(&ret, g_pika_conf->max_cache_statistic_keys());
+    EncodeString(&ret, "small-compaction-threshold");
+    EncodeInt32(&ret, g_pika_conf->small_compaction_threshold());
     EncodeString(&ret, "max-background-flushes");
     EncodeInt32(&ret, g_pika_conf->max_background_flushes());
     EncodeString(&ret, "max-background-compactions");

src/pika_conf.cc

@@ -174,6 +174,19 @@ int PikaConf::Load()
       target_file_size_base_ = 1048576; // 10M
   }
 
+  max_cache_statistic_keys_ = 0;
+  GetConfInt("max-cache-statistic-keys", &max_cache_statistic_keys_);
+  if (max_cache_statistic_keys_ <= 0) {
+    max_cache_statistic_keys_ = 0;
+  }
+
+  small_compaction_threshold_ = 5000;
+  GetConfInt("small-compaction-threshold", &small_compaction_threshold_);
+  if (small_compaction_threshold_ <= 0
+    || small_compaction_threshold_ >= 100000) {
+    small_compaction_threshold_ = 5000;
+  }
+
   max_background_flushes_ = 1;
   GetConfInt("max-background-flushes", &max_background_flushes_);
   if (max_background_flushes_ <= 0) {
@@ -316,6 +329,8 @@ int PikaConf::ConfigRewrite() {
   SetConfInt("binlog-file-size", binlog_file_size_);
   SetConfStr("identify-binlog-type", identify_binlog_type_);
   SetConfStr("compression", compression_);
+  SetConfInt("max-cache-statistic-keys", max_cache_statistic_keys_);
+  SetConfInt("small-compaction-threshold", small_compaction_threshold_);
   SetConfInt("max-background-flushes", max_background_flushes_);
   SetConfInt("max-background-compactions", max_background_compactions_);
   SetConfInt("max-cache-files", max_cache_files_);

src/pika_server.cc

@@ -264,6 +264,8 @@ void PikaServer::RocksdbOptionInit(blackwidow::BlackwidowOptions* bw_option) {
   bw_option->table_options.cache_index_and_filter_blocks = g_pika_conf->cache_index_and_filter_blocks();
   bw_option->block_cache_size = g_pika_conf->block_cache();
   bw_option->share_block_cache = g_pika_conf->share_block_cache();
+  bw_option->statistics_max_size = g_pika_conf->max_cache_statistic_keys();
+  bw_option->small_compaction_threshold = g_pika_conf->small_compaction_threshold();
 }
 
 void PikaServer::Start() {