support merge and delete in file ingestion

Summary: Previously sst_file_writer only supports kTypeValue, we need kTypeMerge and kTypeDeletion also as user requested. Closes facebook#2361 Differential Revision: D5139402 Pulled By: lightmark fbshipit-source-id: 092a60756d01692539d817a3765ebfd58a8d7f88
hihain · May 26, 2017 · f7bb1a0 · f7bb1a0
1 parent c2c62ad
commit f7bb1a0
Show file tree

Hide file tree

Showing 11 changed files with 527 additions and 155 deletions.
diff --git a/db/c.cc b/db/c.cc
@@ -2852,7 +2852,26 @@ void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer,
 void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key,
                                size_t keylen, const char* val, size_t vallen,
                                char** errptr) {
-  SaveError(errptr, writer->rep->Add(Slice(key, keylen), Slice(val, vallen)));
+  SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_sstfilewriter_put(rocksdb_sstfilewriter_t* writer, const char* key,
+                               size_t keylen, const char* val, size_t vallen,
+                               char** errptr) {
+  SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_sstfilewriter_merge(rocksdb_sstfilewriter_t* writer,
+                                 const char* key, size_t keylen,
+                                 const char* val, size_t vallen,
+                                 char** errptr) {
+  SaveError(errptr, writer->rep->Merge(Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_sstfilewriter_delete(rocksdb_sstfilewriter_t* writer,
+                                  const char* key, size_t keylen,
+                                  char** errptr) {
+  SaveError(errptr, writer->rep->Delete(Slice(key, keylen)));
 }
 
 void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,

diff --git a/db/c_test.c b/db/c_test.c
@@ -569,11 +569,11 @@ int main(int argc, char** argv) {
     unlink(sstfilename);
     rocksdb_sstfilewriter_open(writer, sstfilename, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk1", 5, "v1", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v2", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v3", 2, &err);
     CheckNoError(err);
     rocksdb_sstfilewriter_finish(writer, &err);
     CheckNoError(err);
@@ -590,11 +590,11 @@ int main(int argc, char** argv) {
     unlink(sstfilename);
     rocksdb_sstfilewriter_open(writer, sstfilename, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v4", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk22", 6, "v5", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v6", 2, &err);
     CheckNoError(err);
     rocksdb_sstfilewriter_finish(writer, &err);
     CheckNoError(err);

diff --git a/db/db_merge_operator_test.cc b/db/db_merge_operator_test.cc
@@ -20,28 +20,6 @@ class DBMergeOperatorTest : public DBTestBase {
   DBMergeOperatorTest() : DBTestBase("/db_merge_operator_test") {}
 };
 
-// A test merge operator mimics put but also fails if one of merge operands is
-// "corrupted".
-class TestPutOperator : public MergeOperator {
- public:
-  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
-                           MergeOperationOutput* merge_out) const override {
-    if (merge_in.existing_value != nullptr &&
-        *(merge_in.existing_value) == "corrupted") {
-      return false;
-    }
-    for (auto value : merge_in.operand_list) {
-      if (value == "corrupted") {
-        return false;
-      }
-    }
-    merge_out->existing_operand = merge_in.operand_list.back();
-    return true;
-  }
-
-  virtual const char* Name() const override { return "TestPutOperator"; }
-};
-
 TEST_F(DBMergeOperatorTest, MergeErrorOnRead) {
   Options options;
   options.create_if_missing = true;

diff --git a/db/db_test_util.h b/db/db_test_util.h
@@ -586,6 +586,28 @@ class OnFileDeletionListener : public EventListener {
 };
 #endif
 
+// A test merge operator mimics put but also fails if one of merge operands is
+// "corrupted".
+class TestPutOperator : public MergeOperator {
+ public:
+  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
+                           MergeOperationOutput* merge_out) const override {
+    if (merge_in.existing_value != nullptr &&
+        *(merge_in.existing_value) == "corrupted") {
+      return false;
+    }
+    for (auto value : merge_in.operand_list) {
+      if (value == "corrupted") {
+        return false;
+      }
+    }
+    merge_out->existing_operand = merge_in.operand_list.back();
+    return true;
+  }
+
+  virtual const char* Name() const override { return "TestPutOperator"; }
+};
+
 class DBTestBase : public testing::Test {
  protected:
   // Sequence of option configurations to try