Skip to content

Commit

Permalink
Clean up compression logging
Browse files Browse the repository at this point in the history 
Summary: Now we add warnings when user configures compression and the compression is not supported.

Test Plan:
Configured compression to non-supported values. Observed messages in my log:

    2015/03/26-12:17:57.586341 7ffb8a496840 [WARN] Compression type chosen for level 2 is not supported: LZ4. RocksDB will not compress data on level 2.

    2015/03/26-12:19:10.768045 7f36f15c5840 [WARN] Compression type chosen is not supported: LZ4. RocksDB will not compress data.

Reviewers: rven, sdong, yhchiang

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D35979
  • Loading branch information
@igorcanadi
igorcanadi committed Apr 6, 2015
1 parent e3ee98b commit 5e067a7
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 147 deletions.
19 changes: 18 additions & 1 deletion db/column_family.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
result.min_write_buffer_number_to_merge =
std::min(result.min_write_buffer_number_to_merge,
result.max_write_buffer_number - 1);
result.compression_per_level = src.compression_per_level;
if (result.max_mem_compaction_level >= result.num_levels) {
result.max_mem_compaction_level = result.num_levels - 1;
}
Expand All @@ -155,6 +154,24 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
}
}

if (!src.compression_per_level.empty()) {
for (size_t level = 0; level < src.compression_per_level.size(); ++level) {
if (!CompressionTypeSupported(src.compression_per_level[level])) {
Log(InfoLogLevel::WARN_LEVEL, db_options.info_log,
"Compression type chosen for level %zu is not supported: %s. "
"RocksDB "
"will not compress data on level %zu.",
level, CompressionTypeToString(src.compression_per_level[level]),
level);
}
}
} else if (!CompressionTypeSupported(src.compression)) {
Log(InfoLogLevel::WARN_LEVEL, db_options.info_log,
"Compression type chosen is not supported: %s. RocksDB will not "
"compress data.",
CompressionTypeToString(src.compression));
}

if (result.compaction_style == kCompactionStyleFIFO) {
result.num_levels = 1;
// since we delete level0 files in FIFO compaction when there are too many
Expand Down
29 changes: 9 additions & 20 deletions db/db_impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
#include "util/autovector.h"
#include "util/build_version.h"
#include "util/coding.h"
#include "util/compression.h"
#include "util/db_info_dumper.h"
#include "util/file_util.h"
#include "util/hash_skiplist_rep.h"
Expand Down Expand Up @@ -189,26 +190,14 @@ CompressionType GetCompressionFlush(const ImmutableCFOptions& ioptions) {
}

void DumpCompressionInfo(Logger* logger) {
Log(InfoLogLevel::INFO_LEVEL, logger, "Snappy "
#ifndef SNAPPY
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "Zlib "
#ifndef ZLIB
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "Bzip "
#ifndef BZIP2
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "LZ4 "
#ifndef LZ4
"NOT "
#endif
"supported");
Log(InfoLogLevel::INFO_LEVEL, logger, "Compression algorithms supported:");
Log(InfoLogLevel::INFO_LEVEL, logger, "\tSnappy supported: %d",
Snappy_Supported());
Log(InfoLogLevel::INFO_LEVEL, logger, "\tZlib supported: %d",
Zlib_Supported());
Log(InfoLogLevel::INFO_LEVEL, logger, "\tBzip supported: %d",
BZip2_Supported());
Log(InfoLogLevel::INFO_LEVEL, logger, "\tLZ4 supported: %d", LZ4_Supported());
}

} // namespace
Expand Down
60 changes: 9 additions & 51 deletions db/db_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,41 +60,6 @@

namespace rocksdb {

static bool SnappyCompressionSupported(const CompressionOptions& options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Snappy_Compress(options, in.data(), in.size(), &out);
}

static bool SnappyCompressionSupported() {
CompressionOptions options;
return SnappyCompressionSupported(options);
}

static bool ZlibCompressionSupported(const CompressionOptions& options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Zlib_Compress(options, 2, in.data(), in.size(), &out);
}

static bool BZip2CompressionSupported(const CompressionOptions& options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return BZip2_Compress(options, 2, in.data(), in.size(), &out);
}

static bool LZ4CompressionSupported(const CompressionOptions &options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4_Compress(options, 2, in.data(), in.size(), &out);
}

static bool LZ4HCCompressionSupported(const CompressionOptions &options) {
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4HC_Compress(options, 2, in.data(), in.size(), &out);
}

static std::string RandomString(Random* rnd, int len) {
std::string r;
test::RandomString(rnd, len, &r);
Expand Down Expand Up @@ -4418,7 +4383,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
}

TEST_F(DBTest, CompressedCache) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
int num_iter = 80;
Expand Down Expand Up @@ -4542,7 +4507,7 @@ static std::string CompressibleString(Random* rnd, int len) {
}

TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}

Expand Down Expand Up @@ -4611,7 +4576,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
}

TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
Options options;
Expand Down Expand Up @@ -4655,7 +4620,7 @@ TEST_F(DBTest, FailMoreDbPaths) {
}

TEST_F(DBTest, UniversalCompactionSecondPathRatio) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
Options options;
Expand Down Expand Up @@ -5348,25 +5313,18 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
options.level0_file_num_compaction_trigger = 3;
options.create_if_missing = true;

if (SnappyCompressionSupported(CompressionOptions(wbits, lev, strategy))) {
if (Snappy_Supported()) {
type = kSnappyCompression;
fprintf(stderr, "using snappy\n");
} else if (ZlibCompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
} else if (Zlib_Supported()) {
type = kZlibCompression;
fprintf(stderr, "using zlib\n");
} else if (BZip2CompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
} else if (BZip2_Supported()) {
type = kBZip2Compression;
fprintf(stderr, "using bzip2\n");
} else if (LZ4CompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
} else if (LZ4_Supported()) {
type = kLZ4Compression;
fprintf(stderr, "using lz4\n");
} else if (LZ4HCCompressionSupported(
CompressionOptions(wbits, lev, strategy))) {
type = kLZ4HCCompression;
fprintf(stderr, "using lz4hc\n");
} else {
fprintf(stderr, "skipping test, compression disabled\n");
return false;
Expand Down Expand Up @@ -11129,7 +11087,7 @@ TEST_F(DBTest, DynamicLevelMaxBytesBase2) {
}

TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
return;
}
const int kNKeys = 120;
Expand Down
6 changes: 6 additions & 0 deletions include/rocksdb/options.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ enum CompressionType : char {
kBZip2Compression = 0x3, kLZ4Compression = 0x4, kLZ4HCCompression = 0x5
};

// returns true if RocksDB was correctly linked with compression library and
// supports the compression type
extern bool CompressionTypeSupported(CompressionType compression_type);
// Returns a human-readable name of the compression type
extern const char* CompressionTypeToString(CompressionType compression_type);

enum CompactionStyle : char {
// level based compaction style
kCompactionStyleLevel = 0x0,
Expand Down
80 changes: 9 additions & 71 deletions table/table_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,61 +537,6 @@ class DBConstructor: public Constructor {
DB* db_;
};

static bool SnappyCompressionSupported() {
#ifdef SNAPPY
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Snappy_Compress(Options().compression_opts, in.data(), in.size(),
&out);
#else
return false;
#endif
}

static bool ZlibCompressionSupported() {
#ifdef ZLIB
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return Zlib_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}

static bool BZip2CompressionSupported() {
#ifdef BZIP2
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return BZip2_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}

static bool LZ4CompressionSupported() {
#ifdef LZ4
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}

static bool LZ4HCCompressionSupported() {
#ifdef LZ4
std::string out;
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
return LZ4HC_Compress(Options().compression_opts, 2, in.data(), in.size(),
&out);
#else
return false;
#endif
}

enum TestType {
BLOCK_BASED_TABLE_TEST,
PLAIN_TABLE_SEMI_FIXED_PREFIX,
Expand Down Expand Up @@ -623,22 +568,20 @@ static std::vector<TestArgs> GenerateArgList() {
// Only add compression if it is supported
std::vector<std::pair<CompressionType, bool>> compression_types;
compression_types.emplace_back(kNoCompression, false);
if (SnappyCompressionSupported()) {
if (Snappy_Supported()) {
compression_types.emplace_back(kSnappyCompression, false);
}
if (ZlibCompressionSupported()) {
if (Zlib_Supported()) {
compression_types.emplace_back(kZlibCompression, false);
compression_types.emplace_back(kZlibCompression, true);
}
if (BZip2CompressionSupported()) {
if (BZip2_Supported()) {
compression_types.emplace_back(kBZip2Compression, false);
compression_types.emplace_back(kBZip2Compression, true);
}
if (LZ4CompressionSupported()) {
if (LZ4_Supported()) {
compression_types.emplace_back(kLZ4Compression, false);
compression_types.emplace_back(kLZ4Compression, true);
}
if (LZ4HCCompressionSupported()) {
compression_types.emplace_back(kLZ4HCCompression, false);
compression_types.emplace_back(kLZ4HCCompression, true);
}
Expand Down Expand Up @@ -1918,36 +1861,31 @@ static void DoCompressionTest(CompressionType comp) {

TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) {
std::vector<CompressionType> compression_state;
if (!SnappyCompressionSupported()) {
if (!Snappy_Supported()) {
fprintf(stderr, "skipping snappy compression tests\n");
} else {
compression_state.push_back(kSnappyCompression);
}

if (!ZlibCompressionSupported()) {
if (!Zlib_Supported()) {
fprintf(stderr, "skipping zlib compression tests\n");
} else {
compression_state.push_back(kZlibCompression);
}

// TODO(kailiu) DoCompressionTest() doesn't work with BZip2.
/*
if (!BZip2CompressionSupported()) {
if (!BZip2_Supported()) {
fprintf(stderr, "skipping bzip2 compression tests\n");
} else {
compression_state.push_back(kBZip2Compression);
}
*/

if (!LZ4CompressionSupported()) {
fprintf(stderr, "skipping lz4 compression tests\n");
if (!LZ4_Supported()) {
fprintf(stderr, "skipping lz4 and lz4hc compression tests\n");
} else {
compression_state.push_back(kLZ4Compression);
}

if (!LZ4HCCompressionSupported()) {
fprintf(stderr, "skipping lz4hc compression tests\n");
} else {
compression_state.push_back(kLZ4HCCompression);
}

Expand Down
28 changes: 28 additions & 0 deletions util/compression.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,34 @@

namespace rocksdb {

inline bool Snappy_Supported() {
#ifdef SNAPPY
return true;
#endif
return false;
}

inline bool Zlib_Supported() {
#ifdef ZLIB
return true;
#endif
return false;
}

inline bool BZip2_Supported() {
#ifdef BZIP2
return true;
#endif
return false;
}

inline bool LZ4_Supported() {
#ifdef LZ4
return true;
#endif
return false;
}

// compress_format_version can have two values:
// 1 -- decompressed sizes for BZip2 and Zlib are not included in the compressed
// block. Also, decompressed sizes for LZ4 are encoded in platform-dependent
Expand Down
Loading

0 comments on commit 5e067a7

Please sign in to comment.