reverting disastrous MOE commit, returning to r21

git-svn-id: https://leveldb.googlecode.com/svn/trunk@23 62dab493-f737-651d-591e-8d6aee1b9529

leveldb/Makefile → Makefile

@@ -27,6 +27,7 @@ LIBOBJECTS = \
 	./db/version_set.o \
 	./db/write_batch.o \
 	./port/port_posix.o \
+	./port/sha1_portable.o \
 	./table/block.o \
 	./table/block_builder.o \
 	./table/format.o \
@@ -62,6 +63,7 @@ TESTS = \
 	env_test \
 	filename_test \
 	log_test \
+	sha1_test \
 	skiplist_test \
 	table_test \
 	version_edit_test \
@@ -113,6 +115,9 @@ log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
 table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CC) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
 
+sha1_test: port/sha1_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CC) $(LDFLAGS) port/sha1_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
+
 skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CC) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
 

leveldb/README → README

@@ -2,10 +2,10 @@ leveldb: A key-value store
 Authors: Sanjay Ghemawat ([email protected]) and Jeff Dean ([email protected])
 
 The code under this directory implements a system for maintaining a
-persistent key/value store.
+persistent key/value store.  
 
 See doc/index.html for more explanation.
-See doc/impl.html for a brief overview of the implementation.
+See doc/db_layout.txt for a brief overview of the implementation.
 
 The public interface is in include/*.h.  Callers should not include or
 rely on the details of any other header files in this package.  Those

leveldb/TODO → TODO

@@ -8,7 +8,7 @@ db
   object stores, etc. can be done in the background anyway, so
   probably not that important.
 
-api changes:
-- Make it wrappable
+api changes?
+- Efficient large value reading and writing
 
 Faster Get implementation

leveldb/db/builder.cc → db/builder.cc

@@ -38,6 +38,15 @@ Status BuildTable(const std::string& dbname,
     for (; iter->Valid(); iter->Next()) {
       Slice key = iter->key();
       meta->largest.DecodeFrom(key);
+      if (ExtractValueType(key) == kTypeLargeValueRef) {
+        if (iter->value().size() != LargeValueRef::ByteSize()) {
+          s = Status::Corruption("invalid indirect reference hash value (L0)");
+          break;
+        }
+        edit->AddLargeValueRef(LargeValueRef::FromRef(iter->value()),
+                               meta->number,
+                               iter->key());
+      }
       builder->Add(key, iter->value());
     }
 

leveldb/db/builder.h → db/builder.h

@@ -20,9 +20,9 @@ class VersionEdit;
 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to meta->number.  On success, the rest of
 // *meta will be filled with metadata about the generated table, and
-// the file information will be added to *edit.  If no data is present
-// in *iter, meta->file_size will be set to zero, and no Table file
-// will be produced.
+// large value refs and the added file information will be added to
+// *edit.  If no data is present in *iter, meta->file_size will be set
+// to zero, and no Table file will be produced.
 extern Status BuildTable(const std::string& dbname,
                          Env* env,
                          const Options& options,

leveldb/db/corruption_test.cc → db/corruption_test.cc

@@ -121,10 +121,11 @@ class CorruptionTest {
     std::vector<std::string> filenames;
     ASSERT_OK(env_.GetChildren(dbname_, &filenames));
     uint64_t number;
+    LargeValueRef large_ref;
     FileType type;
     std::vector<std::string> candidates;
     for (int i = 0; i < filenames.size(); i++) {
-      if (ParseFileName(filenames[i], &number, &type) &&
+      if (ParseFileName(filenames[i], &number, &large_ref, &type) &&
           type == filetype) {
         candidates.push_back(dbname_ + "/" + filenames[i]);
       }
@@ -275,6 +276,29 @@ TEST(CorruptionTest, SequenceNumberRecovery) {
   ASSERT_EQ("v6", v);
 }
 
+TEST(CorruptionTest, LargeValueRecovery) {
+  Options options;
+  options.large_value_threshold = 10000;
+  Reopen(&options);
+
+  Random rnd(301);
+  std::string big;
+  ASSERT_OK(db_->Put(WriteOptions(),
+                     "foo", test::RandomString(&rnd, 100000, &big)));
+  std::string v;
+  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+  ASSERT_EQ(big, v);
+
+  RepairDB();
+  Reopen();
+  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+  ASSERT_EQ(big, v);
+
+  Reopen();
+  ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+  ASSERT_EQ(big, v);
+}
+
 TEST(CorruptionTest, CorruptedDescriptor) {
   ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);

leveldb/db/db_bench.cc → db/db_bench.cc

@@ -28,6 +28,7 @@
 //      readreverse   -- read N values in reverse order
 //      readrandom    -- read N values in random order
 //      crc32c        -- repeated crc32c of 4K of data
+//      sha1          -- repeated SHA1 computation over 4K of data
 //   Meta operations:
 //      compact     -- Compact the entire DB
 //      stats       -- Print DB stats
@@ -47,6 +48,7 @@ static const char* FLAGS_benchmarks =
     "readreverse,"
     "fill100K,"
     "crc32c,"
+    "sha1,"
     "snappycomp,"
     "snappyuncomp,"
     ;
@@ -364,6 +366,8 @@ class Benchmark {
         Compact();
       } else if (name == Slice("crc32c")) {
         Crc32c(4096, "(4K per op)");
+      } else if (name == Slice("sha1")) {
+        SHA1(4096, "(4K per op)");
       } else if (name == Slice("snappycomp")) {
         SnappyCompress();
       } else if (name == Slice("snappyuncomp")) {
@@ -402,6 +406,24 @@ class Benchmark {
     message_ = label;
   }
 
+  void SHA1(int size, const char* label) {
+    // SHA1 about 100MB of data total
+    std::string data(size, 'x');
+    int64_t bytes = 0;
+    char sha1[20];
+    while (bytes < 100 * 1048576) {
+      port::SHA1_Hash(data.data(), size, sha1);
+      FinishedSingleOp();
+      bytes += size;
+    }
+
+    // Print so result is not dead
+    fprintf(stderr, "... sha1=%02x...\r", static_cast<unsigned int>(sha1[0]));
+
+    bytes_ = bytes;
+    message_ = label;
+  }
+
   void SnappyCompress() {
     Slice input = gen_.Generate(Options().block_size);
     int64_t bytes = 0;