forked from facebook/rocksdb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Summary:
The primary motivation of the changes is to make it easier to figure out the inside of the tables.
* rename "table stats" to "table properties" since now we have more than "integers" to store in the property block.
* Add filter block size to the basic table properties.
* Whenever a table is built, we'll log the table properties (the sample output is in Test Plan).
* Make an api to expose deleted keys.
Test Plan:
Passed all existing test. and the sample output of table stats:
==================================================================
Basic Properties
------------------------------------------------------------------
# data blocks: 1
# entries: 1
raw key size: 9
raw average key size: 9
raw value size: 9
raw average value size: 0
data block size: 25
index block size: 27
filter block size: 18
(estimated) table size: 70
filter policy: rocksdb.BuiltinBloomFilter
==================================================================
User collected properties: InternalKeyPropertiesCollector
------------------------------------------------------------------
kDeletedKeys: 1
==================================================================
Reviewers: dhruba, haobo
Reviewed By: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14187- Loading branch information
Showing
17 changed files
with
590 additions
and
373 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| // Copyright (c) 2013, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| #include "db/table_properties_collector.h" | ||
|
|
||
| #include "db/dbformat.h" | ||
| #include "util/coding.h" | ||
|
|
||
| namespace rocksdb { | ||
|
|
||
| namespace { | ||
| void AppendProperty( | ||
| std::string& props, | ||
| const std::string& key, | ||
| const std::string& value, | ||
| const std::string& prop_delim, | ||
| const std::string& kv_delim) { | ||
| props.append(key); | ||
| props.append(kv_delim); | ||
| props.append(value); | ||
| props.append(prop_delim); | ||
| } | ||
|
|
||
| template <class TValue> | ||
| void AppendProperty( | ||
| std::string& props, | ||
| const std::string& key, | ||
| const TValue& value, | ||
| const std::string& prop_delim, | ||
| const std::string& kv_delim) { | ||
| AppendProperty( | ||
| props, key, std::to_string(value), prop_delim, kv_delim | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| std::string TableProperties::ToString( | ||
| const std::string& prop_delim, | ||
| const std::string& kv_delim) const { | ||
| std::string result; | ||
| result.reserve(1024); | ||
|
|
||
| // Basic Info | ||
| AppendProperty( | ||
| result, "# data blocks", num_data_blocks, prop_delim, kv_delim | ||
| ); | ||
| AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); | ||
|
|
||
| AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); | ||
| AppendProperty( | ||
| result, | ||
| "raw average key size", | ||
| num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, | ||
| prop_delim, | ||
| kv_delim | ||
| ); | ||
| AppendProperty( | ||
| result, "raw value size", raw_value_size, prop_delim, kv_delim | ||
| ); | ||
| AppendProperty( | ||
| result, | ||
| "raw average value size", | ||
| num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, | ||
| prop_delim, | ||
| kv_delim | ||
| ); | ||
|
|
||
| AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); | ||
| AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); | ||
| AppendProperty( | ||
| result, "filter block size", filter_size, prop_delim, kv_delim | ||
| ); | ||
| AppendProperty( | ||
| result, | ||
| "(estimated) table size=", | ||
| data_size + index_size + filter_size, | ||
| prop_delim, | ||
| kv_delim | ||
| ); | ||
|
|
||
| AppendProperty( | ||
| result, | ||
| "filter policy name", | ||
| filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, | ||
| prop_delim, | ||
| kv_delim | ||
| ); | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| Status InternalKeyPropertiesCollector::Add( | ||
| const Slice& key, const Slice& value) { | ||
| ParsedInternalKey ikey; | ||
| if (!ParseInternalKey(key, &ikey)) { | ||
| return Status::InvalidArgument("Invalid internal key"); | ||
| } | ||
|
|
||
| if (ikey.type == ValueType::kTypeDeletion) { | ||
| ++deleted_keys_; | ||
| } | ||
|
|
||
| return Status::OK(); | ||
| } | ||
|
|
||
| Status InternalKeyPropertiesCollector::Finish( | ||
| TableProperties::UserCollectedProperties* properties) { | ||
| assert(properties); | ||
| assert(properties->find( | ||
| InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end()); | ||
| std::string val; | ||
|
|
||
| PutVarint64(&val, deleted_keys_); | ||
| properties->insert({ InternalKeyTablePropertiesNames::kDeletedKeys, val }); | ||
|
|
||
| return Status::OK(); | ||
| } | ||
|
|
||
| TableProperties::UserCollectedProperties | ||
| InternalKeyPropertiesCollector::GetReadableProperties() const { | ||
| return { | ||
| { "kDeletedKeys", std::to_string(deleted_keys_) } | ||
| }; | ||
| } | ||
|
|
||
|
|
||
| Status UserKeyTablePropertiesCollector::Add( | ||
| const Slice& key, const Slice& value) { | ||
| ParsedInternalKey ikey; | ||
| if (!ParseInternalKey(key, &ikey)) { | ||
| return Status::InvalidArgument("Invalid internal key"); | ||
| } | ||
|
|
||
| return collector_->Add(ikey.user_key, value); | ||
| } | ||
|
|
||
| Status UserKeyTablePropertiesCollector::Finish( | ||
| TableProperties::UserCollectedProperties* properties) { | ||
| return collector_->Finish(properties); | ||
| } | ||
|
|
||
| TableProperties::UserCollectedProperties | ||
| UserKeyTablePropertiesCollector::GetReadableProperties() const { | ||
| return collector_->GetReadableProperties(); | ||
| } | ||
|
|
||
|
|
||
| const std::string InternalKeyTablePropertiesNames::kDeletedKeys | ||
| = "rocksdb.deleted.keys"; | ||
|
|
||
| uint64_t GetDeletedKeys( | ||
| const TableProperties::UserCollectedProperties& props) { | ||
| auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys); | ||
| if (pos == props.end()) { | ||
| return 0; | ||
| } | ||
| Slice raw = pos->second; | ||
| uint64_t val = 0; | ||
| return GetVarint64(&raw, &val) ? val : 0; | ||
| } | ||
|
|
||
| } // namespace rocksdb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| // Copyright (c) 2013, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
| // | ||
| // This file defines a collection of statistics collectors. | ||
| #pragma once | ||
|
|
||
| #include "rocksdb/table_properties.h" | ||
|
|
||
| #include <memory> | ||
| #include <string> | ||
| #include <vector> | ||
|
|
||
| namespace rocksdb { | ||
|
|
||
| struct InternalKeyTablePropertiesNames { | ||
| static const std::string kDeletedKeys; | ||
| }; | ||
|
|
||
| // Collecting the statistics for internal keys. Visible only by internal | ||
| // rocksdb modules. | ||
| class InternalKeyPropertiesCollector : public TablePropertiesCollector { | ||
| public: | ||
| virtual Status Add(const Slice& key, const Slice& value) override; | ||
|
|
||
| virtual Status Finish( | ||
| TableProperties::UserCollectedProperties* properties) override; | ||
|
|
||
| virtual const char* Name() const override { | ||
| return "InternalKeyPropertiesCollector"; | ||
| } | ||
|
|
||
| TableProperties::UserCollectedProperties | ||
| GetReadableProperties() const override; | ||
|
|
||
| private: | ||
| uint64_t deleted_keys_ = 0; | ||
| }; | ||
|
|
||
| // When rocksdb creates a new table, it will encode all "user keys" into | ||
| // "internal keys", which contains meta information of a given entry. | ||
| // | ||
| // This class extracts user key from the encoded internal key when Add() is | ||
| // invoked. | ||
| class UserKeyTablePropertiesCollector : public TablePropertiesCollector { | ||
| public: | ||
| explicit UserKeyTablePropertiesCollector( | ||
| TablePropertiesCollector* collector) : | ||
| UserKeyTablePropertiesCollector( | ||
| std::shared_ptr<TablePropertiesCollector>(collector) | ||
| ) { | ||
| } | ||
|
|
||
| explicit UserKeyTablePropertiesCollector( | ||
| std::shared_ptr<TablePropertiesCollector> collector) : | ||
| collector_(collector) { | ||
| } | ||
|
|
||
| virtual ~UserKeyTablePropertiesCollector() { } | ||
|
|
||
| virtual Status Add(const Slice& key, const Slice& value) override; | ||
|
|
||
| virtual Status Finish( | ||
| TableProperties::UserCollectedProperties* properties) override; | ||
|
|
||
| virtual const char* Name() const override { return collector_->Name(); } | ||
|
|
||
| TableProperties::UserCollectedProperties | ||
| GetReadableProperties() const override; | ||
|
|
||
| protected: | ||
| std::shared_ptr<TablePropertiesCollector> collector_; | ||
| }; | ||
|
|
||
| } // namespace rocksdb |
Oops, something went wrong.