Skip to content

Commit

Permalink
Commandline tool to compace LevelDB databases.
Browse files Browse the repository at this point in the history 
Summary:
A simple CLI which calles DB->CompactRange()
Can take String key's as range.

Test Plan:
Inserted data into a table.
Waited for a minute, used compact tool on it. File modification time's
changed so Compact did something on the files.

Existing unit tests work.

Reviewers: heyongqiang, dhruba

Reviewed By: dhruba

Differential Revision: https://reviews.facebook.net/D5697
  • Loading branch information
Abhishek Kona committed Oct 1, 2012
1 parent a321d5b commit fec8131
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 65 deletions.
16 changes: 8 additions & 8 deletions db/db_bench.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ class RandomGenerator {
pos_ += len;
return Slice(data_.data() + pos_ - len, len);
}
};
};
static Slice TrimSpace(Slice s) {
int start = 0;
while (start < s.size() && isspace(s[start])) {
Expand Down Expand Up @@ -515,9 +515,9 @@ class Benchmark {

public:
Benchmark()
: cache_(FLAGS_cache_size >= 0 ?
(FLAGS_cache_numshardbits >= 1 ?
NewLRUCache(FLAGS_cache_size, FLAGS_cache_numshardbits) :
: cache_(FLAGS_cache_size >= 0 ?
(FLAGS_cache_numshardbits >= 1 ?
NewLRUCache(FLAGS_cache_size, FLAGS_cache_numshardbits) :
NewLRUCache(FLAGS_cache_size)) : NULL),
filter_policy_(FLAGS_bloom_bits >= 0
? NewBloomFilterPolicy(FLAGS_bloom_bits)
Expand All @@ -528,9 +528,9 @@ class Benchmark {
entries_per_batch_(1),
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
writes_(FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes),
readwrites_((FLAGS_writes < 0 && FLAGS_reads < 0)? FLAGS_num :
readwrites_((FLAGS_writes < 0 && FLAGS_reads < 0)? FLAGS_num :
((FLAGS_writes > FLAGS_reads) ? FLAGS_writes : FLAGS_reads)
),
),
heap_counter_(0) {
std::vector<std::string> files;
FLAGS_env->GetChildren(FLAGS_db, &files);
Expand Down Expand Up @@ -1048,7 +1048,7 @@ class Benchmark {

//
// This is diffferent from ReadWhileWriting because it does not use
// an extra thread.
// an extra thread.
//
void ReadRandomWriteRandom(ThreadState* thread) {
ReadOptions options(FLAGS_verify_checksum, true);
Expand Down Expand Up @@ -1090,7 +1090,7 @@ class Benchmark {
thread->stats.FinishedSingleOp();
}
char msg[100];
snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld )",
snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld )",
reads_done, writes_done, readwrites_);
thread->stats.AddMessage(msg);
}
Expand Down
202 changes: 145 additions & 57 deletions tools/ldb.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@
#include <sstream>

#include "leveldb/db.h"
#include "leveldb/options.h"
#include "leveldb/iterator.h"
#include "leveldb/slice.h"

std::string HexToString(std::string str) {
std::string HexToString(const std::string& str) {
std::string parsed;
for (int i = 0; i < str.length(); ) {
int c;
Expand All @@ -20,73 +22,58 @@ std::string HexToString(std::string str) {
return parsed;
}

static void print_help() {

static void print_usage() {
fprintf(stderr,
"db_dump "
"--start=[START_KEY] "
"--end=[END_KEY] "
"--max_keys=[NUM] "
"--hex "
"--count_only "
"--stats "
"[PATH]\n");
"ldb [compact|dump] "
"--db-path=database_path "
"[--from=START KEY] "
"[--to=END KEY ] "
"[--max_keys=[NUM] (only for dump)] "
"[--hex ] "
"[--count_only (only for dump)] "
"[--stats (only for dump) ] \n");
}

int main(int argc, char** argv) {
std::string db_path;
std::string start;
std::string end;
uint64_t max_keys = -1;
bool print_stats = false;
bool count_only = false;
uint64_t count = 0;
bool hex = false;

// Parse command line args
char junk;
uint64_t n;
for (int i = 1; i < argc; i++) {
if (strncmp(argv[i], "--start=", 8) == 0) {
start= argv[i] + 8;
} else if (strncmp(argv[i], "--end=", 6) == 0) {
end = argv[i] + 6;
} else if (sscanf(argv[i], "--max_keys=%ld%c", &n, &junk) == 1) {
max_keys = n;
} else if (strncmp(argv[i], "--stats", 7) == 0) {
print_stats = true;
} else if (strncmp(argv[i], "--count_only", 12) == 0) {
count_only = true;
} else if (strncmp(argv[i], "--hex", 5) == 0) {
hex = true;
} else if (i == (argc - 1)) {
db_path = argv[i];
} else {
print_help();
exit(1);
}
static void safe_open_db(const std::string& dbname, leveldb::DB** db) {
leveldb::Options options;
options.create_if_missing = false;
leveldb::Status status = leveldb::DB::Open(options, dbname, db);

if(!status.ok()) {
fprintf(
stderr,
"Could not open db at %s\nERROR: %s",
dbname.data(),
status.ToString().data()
);
exit(1);
}
}


static void dump_db(
const std::string& db_path,
std::string& start,
std::string& end,
int64_t max_keys,
const bool hex,
const bool print_stats,
const bool count_only
) {
// Parse command line args
uint64_t count = 0;


if (hex) {
start = HexToString(start);
end = HexToString(end);
}

if (db_path.empty()) {
print_help();
exit(1);
}

// Open DB
leveldb::Options options;
leveldb::DB *db;
leveldb::Status status = leveldb::DB::Open(options, db_path, &db);
safe_open_db(db_path, &db);

if (!status.ok()) {
fprintf(stderr, "%s\n", status.ToString().c_str());
exit(1);
}

// Print DB stats if desired
if (print_stats) {
std::string stats;
if (db->GetProperty("leveldb.stats", &stats)) {
Expand All @@ -96,7 +83,7 @@ int main(int argc, char** argv) {

// Setup key iterator
leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions());
status = iter->status();
leveldb::Status status = iter->status();
if (!status.ok()) {
fprintf(stderr, "%s\n", status.ToString().c_str());
delete db;
Expand Down Expand Up @@ -140,7 +127,108 @@ int main(int argc, char** argv) {
// Clean up
delete iter;
delete db;
}

return 0;
static void compact(
const std::string dbname,
std::string from,
std::string to,
const bool hex
) {

leveldb::DB* db;
safe_open_db(dbname, &db);

if(hex) {
from = HexToString(from);
to = HexToString(to);
}

leveldb::Slice* begin = from.empty() ? NULL : new leveldb::Slice(from);
leveldb::Slice* end = to.empty() ? NULL : new leveldb::Slice(to);
db->CompactRange(begin, end);
delete db;
}

int main(int argc, char** argv) {

enum {
DUMP, COMPACT
} command;

if (argc < 2) {
print_usage();
exit(1);
}

size_t n;

const std::string dbnameKey = "--db-path=";
const std::string toKey = "--to=";
const std::string fromKey = "--from=";
std::string dbname;
bool dbnameFound = false;
std::string from;
std::string to;
int64_t temp;
int64_t max_keys = -1;

bool print_stats = false;
bool count_only = false;
bool hex = false;
char junk;


std::string commandString = argv[1];
if (commandString == "dump") {
command = DUMP;
} else if (commandString == "compact") {
command = COMPACT;
} else {
print_usage();
exit(1);
}

for (int i = 2; i < argc; i++) {
std::string param(argv[i]);
if ((n = param.find(dbnameKey)) != std::string::npos) {
dbname = param.substr(dbnameKey.size());
dbnameFound = true;
} else if ((n = param.find(fromKey)) != std::string::npos) {
from = param.substr(fromKey.size());
} else if ((n = param.find(toKey)) != std::string::npos) {
to = param.substr(toKey.size());
} else if (sscanf(argv[i], "--max_keys=%ld%c", &temp, &junk) == 1) {
max_keys = temp;
} else if (strncmp(argv[i], "--stats", 7) == 0) {
print_stats = true;
} else if (strncmp(argv[i], "--count_only", 12) == 0) {
count_only = true;
} else if (strncmp(argv[i], "--hex", 5) == 0) {
hex = true;
} else {
print_usage();
exit(1);
}
}

if (!dbnameFound || dbname.empty()) {
fprintf(stderr, "DB path required. See help\n");
print_usage();
exit(1);
}

switch(command) {
case DUMP:
dump_db(dbname, from, to, max_keys, hex, print_stats, count_only);
break;
case COMPACT:
compact(dbname, from, to, hex);
break;
default:
print_usage();
exit(1);
}

return 0;
}

0 comments on commit fec8131

Please sign in to comment.