Added JSON manifest dump option to ldb command

Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.

Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```

**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```

Reviewers: sdong, anthony, yhchiang, igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D41727

db/version_edit.cc

@@ -11,6 +11,7 @@
 
 #include "db/version_set.h"
 #include "util/coding.h"
+#include "util/event_logger.h"
 #include "rocksdb/slice.h"
 
 namespace rocksdb {
@@ -359,7 +360,7 @@ std::string VersionEdit::DebugString(bool hex_key) const {
     AppendNumberTo(&r, prev_log_number_);
   }
   if (has_next_file_number_) {
-    r.append("\n  NextFile: ");
+    r.append("\n  NextFileNumber: ");
     AppendNumberTo(&r, next_file_number_);
   }
   if (has_last_sequence_) {
@@ -404,4 +405,75 @@ std::string VersionEdit::DebugString(bool hex_key) const {
   return r;
 }
 
+std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
+  JSONWriter jw;
+  jw << "EditNumber" << edit_num;
+
+  if (has_comparator_) {
+    jw << "Comparator" << comparator_;
+  }
+  if (has_log_number_) {
+    jw << "LogNumber" << log_number_;
+  }
+  if (has_prev_log_number_) {
+    jw << "PrevLogNumber" << prev_log_number_;
+  }
+  if (has_next_file_number_) {
+    jw << "NextFileNumber" << next_file_number_;
+  }
+  if (has_last_sequence_) {
+    jw << "LastSeq" << last_sequence_;
+  }
+
+  if (!deleted_files_.empty()) {
+    jw << "DeletedFiles";
+    jw.StartArray();
+
+    for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
+         iter != deleted_files_.end();
+         ++iter) {
+      jw.StartArrayedObject();
+      jw << "Level" << iter->first;
+      jw << "FileNumber" << iter->second;
+      jw.EndArrayedObject();
+    }
+
+    jw.EndArray();
+  }
+
+  if (!new_files_.empty()) {
+    jw << "AddedFiles";
+    jw.StartArray();
+
+    for (size_t i = 0; i < new_files_.size(); i++) {
+      jw.StartArrayedObject();
+      jw << "Level" << new_files_[i].first;
+      const FileMetaData& f = new_files_[i].second;
+      jw << "FileNumber" << f.fd.GetNumber();
+      jw << "FileSize" << f.fd.GetFileSize();
+      jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
+      jw << "LargestIKey" << f.largest.DebugString(hex_key);
+      jw.EndArrayedObject();
+    }
+
+    jw.EndArray();
+  }
+
+  jw << "ColumnFamily" << column_family_;
+
+  if (is_column_family_add_) {
+    jw << "ColumnFamilyAdd" << column_family_name_;
+  }
+  if (is_column_family_drop_) {
+    jw << "ColumnFamilyDrop" << column_family_name_;
+  }
+  if (has_max_column_family_) {
+    jw << "MaxColumnFamily" << max_column_family_;
+  }
+
+  jw.EndObject();
+
+  return jw.Get();
+}
+
 }  // namespace rocksdb

db/version_edit.h

@@ -227,6 +227,7 @@ class VersionEdit {
   }
 
   std::string DebugString(bool hex_key = false) const;
+  std::string DebugJSON(int edit_num, bool hex_key = false) const;
 
  private:
   friend class VersionSet;

db/version_set.cc

@@ -2572,7 +2572,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
 }
 
 Status VersionSet::DumpManifest(Options& options, std::string& dscname,
-                                bool verbose, bool hex) {
+                                bool verbose, bool hex, bool json) {
   // Open the specified manifest file.
   unique_ptr<SequentialFile> file;
   Status s = options.env->NewSequentialFile(dscname, &file, env_options_);
@@ -2613,9 +2613,10 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
       }
 
       // Write out each individual edit
-      if (verbose) {
-        printf("*************************Edit[%d] = %s\n",
-                count, edit.DebugString(hex).c_str());
+      if (verbose && !json) {
+        printf("%s\n", edit.DebugString(hex).c_str());
+      } else if (json) {
+        printf("%s\n", edit.DebugJSON(count, hex).c_str());
       }
       count++;
 

db/version_set.h

@@ -572,7 +572,7 @@ class VersionSet {
 
   // printf contents (for debugging)
   Status DumpManifest(Options& options, std::string& manifestFileName,
-                      bool verbose, bool hex = false);
+                      bool verbose, bool hex = false, bool json = false);
 
 #endif  // ROCKSDB_LITE
 

util/event_logger.h

@@ -15,11 +15,11 @@
 
 namespace rocksdb {
 
-// JSONWriter doesn't support objects in arrays yet. There wasn't a need for
-// that.
 class JSONWriter {
  public:
-  JSONWriter() : state_(kExpectKey), first_element_(true) { stream_ << "{"; }
+  JSONWriter() : state_(kExpectKey), first_element_(true), in_array_(false) {
+    stream_ << "{";
+  }
 
   void AddKey(const std::string& key) {
     assert(state_ == kExpectKey);
@@ -59,13 +59,15 @@ class JSONWriter {
   void StartArray() {
     assert(state_ == kExpectValue);
     state_ = kInArray;
+    in_array_ = true;
     stream_ << "[";
     first_element_ = true;
   }
 
   void EndArray() {
     assert(state_ == kInArray);
     state_ = kExpectKey;
+    in_array_ = false;
     stream_ << "]";
     first_element_ = false;
   }
@@ -83,6 +85,21 @@ class JSONWriter {
     first_element_ = false;
   }
 
+  void StartArrayedObject() {
+    assert(state_ == kInArray && in_array_);
+    state_ = kExpectValue;
+    if (!first_element_) {
+      stream_ << ", ";
+    }
+    StartObject();
+  }
+
+  void EndArrayedObject() {
+    assert(in_array_);
+    EndObject();
+    state_ = kInArray;
+  }
+
   std::string Get() const { return stream_.str(); }
 
   JSONWriter& operator<<(const char* val) {
@@ -110,9 +127,11 @@ class JSONWriter {
     kExpectKey,
     kExpectValue,
     kInArray,
+    kInArrayedObject,
   };
   JSONWriterState state_;
   bool first_element_;
+  bool in_array_;
   std::ostringstream stream_;
 };
 

util/ldb_cmd.cc

@@ -527,7 +527,7 @@ void DBLoaderCommand::DoCommand() {
 
 namespace {
 
-void DumpManifestFile(std::string file, bool verbose, bool hex) {
+void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
   Options options;
   EnvOptions sopt;
   std::string dbname("dummy");
@@ -540,7 +540,7 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) {
   WriteController wc(options.delayed_write_rate);
   WriteBuffer wb(options.db_write_buffer_size);
   VersionSet versions(dbname, &options, sopt, tc.get(), &wb, &wc);
-  Status s = versions.DumpManifest(options, file, verbose, hex);
+  Status s = versions.DumpManifest(options, file, verbose, hex, json);
   if (!s.ok()) {
     printf("Error in processing file %s %s\n", file.c_str(),
            s.ToString().c_str());
@@ -550,24 +550,28 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) {
 }  // namespace
 
 const string ManifestDumpCommand::ARG_VERBOSE = "verbose";
-const string ManifestDumpCommand::ARG_PATH    = "path";
+const string ManifestDumpCommand::ARG_JSON = "json";
+const string ManifestDumpCommand::ARG_PATH = "path";
 
 void ManifestDumpCommand::Help(string& ret) {
   ret.append("  ");
   ret.append(ManifestDumpCommand::Name());
   ret.append(" [--" + ARG_VERBOSE + "]");
+  ret.append(" [--" + ARG_JSON + "]");
   ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
   ret.append("\n");
 }
 
 ManifestDumpCommand::ManifestDumpCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
     LDBCommand(options, flags, false,
-               BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX})),
+               BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
     verbose_(false),
+    json_(false),
     path_("")
 {
   verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
+  json_ = IsFlagPresent(flags, ARG_JSON);
 
   map<string, string>::const_iterator itr = options.find(ARG_PATH);
   if (itr != options.end()) {
@@ -623,7 +627,8 @@ void ManifestDumpCommand::DoCommand() {
     printf("Processing Manifest file %s\n", manifestfile.c_str());
   }
 
-  DumpManifestFile(manifestfile, verbose_, is_key_hex_);
+  DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
+
   if (verbose_) {
     printf("Processing Manifest file %s done\n", manifestfile.c_str());
   }
@@ -2025,7 +2030,7 @@ void DBFileDumperCommand::DoCommand() {
   manifest_filename.resize(manifest_filename.size() - 1);
   string manifest_filepath = db_->GetName() + "/" + manifest_filename;
   std::cout << manifest_filepath << std::endl;
-  DumpManifestFile(manifest_filepath, false, false);
+  DumpManifestFile(manifest_filepath, false, false, false);
   std::cout << std::endl;
 
   std::cout << "SST Files" << std::endl;

util/ldb_cmd.h

@@ -507,9 +507,11 @@ class ManifestDumpCommand: public LDBCommand {
 
 private:
   bool verbose_;
+  bool json_;
   string path_;
 
   static const string ARG_VERBOSE;
+  static const string ARG_JSON;
   static const string ARG_PATH;
 };