Added a parameter to limit the maximum space amplification for univer…

…sal compaction.

Summary:
Added a new field called max_size_amplification_ratio in the
CompactionOptionsUniversal structure. This determines the maximum
percentage overhead of space amplification.

The size amplification is defined to be the ratio between the size of
the oldest file to the sum of the sizes of all other files. If the
size amplification exceeds the specified value, then min_merge_width
and max_merge_width are ignored and a full compaction of all files is done.
A value of 10 means that the size a database that stores 100 bytes
of user data could occupy 110 bytes of physical storage.

Test Plan: Unit test DBTest.UniversalCompactionSpaceAmplification added.

Reviewers: haobo, emayanke, xjin

Reviewed By: haobo

CC: leveldb

Differential Revision: https://reviews.facebook.net/D12825

db/db_bench.cc

@@ -155,11 +155,17 @@ static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionSty
 
 // Percentage flexibility while comparing file size
 // (for universal compaction only).
-static int FLAGS_universal_size_ratio = 1;
+static int FLAGS_universal_size_ratio = 0;
 
 // The minimum number of files in a single compaction run
 // (for universal compaction only).
-static int FLAGS_compaction_universal_min_merge_width = 2;
+static int FLAGS_universal_min_merge_width = 0;
+
+// The max number of files to compact in universal style compaction
+static unsigned int FLAGS_universal_max_merge_width = 0;
+
+// The max size amplification for universal style compaction
+static unsigned int FLAGS_universal_max_size_amplification_percent = 0;
 
 // Number of bytes to use as a cache of uncompressed data.
 // Negative means use default settings.
@@ -1185,9 +1191,6 @@ class Benchmark {
       FLAGS_min_write_buffer_number_to_merge;
     options.max_background_compactions = FLAGS_max_background_compactions;
     options.compaction_style = FLAGS_compaction_style;
-    options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio;
-    options.compaction_options_universal.min_merge_width =
-      FLAGS_compaction_universal_min_merge_width;
     options.block_size = FLAGS_block_size;
     options.filter_policy = filter_policy_;
     options.prefix_extractor = FLAGS_use_prefix_blooms ? prefix_extractor_
@@ -1290,6 +1293,24 @@ class Benchmark {
       exit(1);
     }
 
+    // set universal style compaction configurations, if applicable
+    if (FLAGS_universal_size_ratio != 0) {
+      options.compaction_options_universal.size_ratio =
+        FLAGS_universal_size_ratio;
+    }
+    if (FLAGS_universal_min_merge_width != 0) {
+      options.compaction_options_universal.min_merge_width =
+        FLAGS_universal_min_merge_width;
+    }
+    if (FLAGS_universal_max_merge_width != 0) {
+      options.compaction_options_universal.max_merge_width =
+        FLAGS_universal_max_merge_width;
+    }
+    if (FLAGS_universal_max_size_amplification_percent != 0) {
+      options.compaction_options_universal.max_size_amplification_percent =
+        FLAGS_universal_max_size_amplification_percent;
+    }
+
     Status s;
     if(FLAGS_read_only) {
       s = DB::OpenForReadOnly(options, FLAGS_db, &db_);
@@ -2242,10 +2263,6 @@ int main(int argc, char** argv) {
   FLAGS_max_background_compactions =
     leveldb::Options().max_background_compactions;
   FLAGS_compaction_style = leveldb::Options().compaction_style;
-  FLAGS_universal_size_ratio =
-    leveldb::Options().compaction_options_universal.size_ratio;
-  FLAGS_compaction_universal_min_merge_width =
-    leveldb::Options().compaction_options_universal.min_merge_width;
   // Compression test code above refers to FLAGS_block_size
   FLAGS_block_size = leveldb::Options().block_size;
   FLAGS_use_os_buffer = leveldb::EnvOptions().use_os_buffer;
@@ -2315,11 +2332,6 @@ int main(int argc, char** argv) {
       FLAGS_max_background_compactions = n;
     } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
       FLAGS_compaction_style = (leveldb::CompactionStyle)n;
-    } else if (sscanf(argv[i], "--universal_size_ratio=%d%c", &n, &junk) == 1) {
-      FLAGS_universal_size_ratio = n;
-    } else if (sscanf(argv[i], "--universal_min_merge_width=%d%c",
-               &n, &junk) == 1) {
-      FLAGS_compaction_universal_min_merge_width = n;
     } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
       FLAGS_cache_size = l;
     } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
@@ -2525,6 +2537,19 @@ int main(int argc, char** argv) {
     } else if (sscanf(argv[i], "--purge_log_after_memtable_flush=%d%c", &n, &junk)
                == 1 && (n == 0 || n ==1 )) {
       FLAGS_purge_log_after_memtable_flush = n;
+    } else if (sscanf(argv[i], "--universal_size_ratio=%d%c",
+                      &n, &junk) == 1) {
+      FLAGS_universal_size_ratio = n;
+    } else if (sscanf(argv[i], "--universal_min_merge_width=%d%c",
+                      &n, &junk) == 1) {
+      FLAGS_universal_min_merge_width = n;
+    } else if (sscanf(argv[i], "--universal_max_merge_width=%d%c",
+                      &n, &junk) == 1) {
+      FLAGS_universal_max_merge_width = n;
+    } else if (sscanf(argv[i],
+               "--universal_max_size_amplification_percent=%d%c",
+               &n, &junk) == 1) {
+      FLAGS_universal_max_size_amplification_percent = n;
     } else {
       fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
       exit(1);

db/db_test.cc

@@ -1728,6 +1728,43 @@ TEST(DBTest, UniversalCompactionTrigger) {
   }
 }
 
+TEST(DBTest, UniversalCompactionSizeAmplification) {
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.write_buffer_size = 100<<10; //100KB
+  options.level0_file_num_compaction_trigger = 2;
+
+  // Trigger compaction if size amplification exceeds 110%
+  options.compaction_options_universal.
+    max_size_amplification_percent = 110;
+  Reopen(&options);
+
+  Random rnd(301);
+  int key_idx = 0;
+
+  //   Generate two files in Level 0. Both files are approx the same size.
+  for (int num = 0;
+       num < options.level0_file_num_compaction_trigger;
+       num++) {
+    // Write 120KB (12 values, each 10K)
+    for (int i = 0; i < 12; i++) {
+      ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
+      key_idx++;
+    }
+    dbfull()->TEST_WaitForCompactMemTable();
+    ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
+  }
+  ASSERT_EQ(NumTableFilesAtLevel(0), 2);
+
+  // Flush whatever is remaining in memtable. This is typically
+  // small, which should not trigger size ratio based compaction
+  // but will instead trigger size amplification.
+  dbfull()->Flush(FlushOptions());
+
+  // Verify that size amplification did occur
+  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+}
+
 TEST(DBTest, ConvertCompactionStyle) {
   Random rnd(301);
   int max_key_level_insert = 200;