forked from facebook/rocksdb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add CompactOnDeletionCollector in utilities/table_properties_collectors.
Summary: This diff adds CompactOnDeletionCollector in utilities/table_properties_collectors, which applies a sliding window to a sst file and mark this file as need-compaction when it observe enough deletion entries within the consecutive keys covered by the sliding window. Test Plan: compact_on_deletion_collector_test Reviewers: igor, anthony, IslamAbdelRahman, kradhakrishnan, yoshinorim, sdong Reviewed By: sdong Subscribers: maykov, dhruba Differential Revision: https://reviews.facebook.net/D41175
- Loading branch information
Showing
7 changed files
with
412 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| // Copyright (c) 2015, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| #pragma once | ||
| #ifndef ROCKSDB_LITE | ||
| #include <memory> | ||
|
|
||
| #include "rocksdb/table_properties.h" | ||
|
|
||
| namespace rocksdb { | ||
|
|
||
| // Creates a factory of a table property collector that marks a SST | ||
| // file as need-compaction when it observe at least "D" deletion | ||
| // entries in any "N" consecutive entires. | ||
| // | ||
| // @param sliding_window_size "N". Note that this number will be | ||
| // round up to the smallest multilpe of 128 that is no less | ||
| // than the specified size. | ||
| // @param deletion_trigger "D". Note that even when "N" is changed, | ||
| // the specified number for "D" will not be changed. | ||
| extern std::shared_ptr<TablePropertiesCollectorFactory> | ||
| NewCompactOnDeletionCollectorFactory( | ||
| size_t sliding_window_size, | ||
| size_t deletion_trigger); | ||
| } // namespace rocksdb | ||
|
|
||
| #endif // !ROCKSDB_LITE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
utilities/table_properties_collectors/compact_on_deletion_collector.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| // Copyright (c) 2015, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| #ifndef ROCKSDB_LITE | ||
| #include <memory> | ||
|
|
||
| #include "rocksdb/utilities/table_properties_collectors.h" | ||
| #include "utilities/table_properties_collectors/compact_on_deletion_collector.h" | ||
|
|
||
| namespace rocksdb { | ||
|
|
||
| CompactOnDeletionCollector::CompactOnDeletionCollector( | ||
| size_t sliding_window_size, | ||
| size_t deletion_trigger) { | ||
| deletion_trigger_ = deletion_trigger; | ||
|
|
||
| // First, compute the number of keys in each bucket. | ||
| bucket_size_ = | ||
| (sliding_window_size + kNumBuckets - 1) / kNumBuckets; | ||
| assert(bucket_size_ > 0U); | ||
|
|
||
| Reset(); | ||
| } | ||
|
|
||
| void CompactOnDeletionCollector::Reset() { | ||
| for (int i = 0; i < kNumBuckets; ++i) { | ||
| num_deletions_in_buckets_[i] = 0; | ||
| } | ||
| current_bucket_ = 0; | ||
| num_keys_in_current_bucket_ = 0; | ||
| num_deletions_in_observation_window_ = 0; | ||
| need_compaction_ = false; | ||
| } | ||
|
|
||
| // AddUserKey() will be called when a new key/value pair is inserted into the | ||
| // table. | ||
| // @params key the user key that is inserted into the table. | ||
| // @params value the value that is inserted into the table. | ||
| // @params file_size file size up to now | ||
| Status CompactOnDeletionCollector::AddUserKey( | ||
| const Slice& key, const Slice& value, | ||
| EntryType type, SequenceNumber seq, | ||
| uint64_t file_size) { | ||
| if (need_compaction_) { | ||
| // If the output file already needs to be compacted, skip the check. | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| if (num_keys_in_current_bucket_ == bucket_size_) { | ||
| // When the current bucket is full, advance the cursor of the | ||
| // ring buffer to the next bucket. | ||
| current_bucket_ = (current_bucket_ + 1) % kNumBuckets; | ||
|
|
||
| // Update the current count of observed deletion keys by excluding | ||
| // the number of deletion keys in the oldest bucket in the | ||
| // observation window. | ||
| assert(num_deletions_in_observation_window_ >= | ||
| num_deletions_in_buckets_[current_bucket_]); | ||
| num_deletions_in_observation_window_ -= | ||
| num_deletions_in_buckets_[current_bucket_]; | ||
| num_deletions_in_buckets_[current_bucket_] = 0; | ||
| num_keys_in_current_bucket_ = 0; | ||
| } | ||
|
|
||
| num_keys_in_current_bucket_++; | ||
| if (type == kEntryDelete) { | ||
| num_deletions_in_observation_window_++; | ||
| num_deletions_in_buckets_[current_bucket_]++; | ||
| if (num_deletions_in_observation_window_ >= deletion_trigger_) { | ||
| need_compaction_ = true; | ||
| } | ||
| } | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| TablePropertiesCollector* CompactOnDeletionCollectorFactory:: | ||
| CreateTablePropertiesCollector() { | ||
| return new CompactOnDeletionCollector( | ||
| sliding_window_size_, deletion_trigger_); | ||
| } | ||
|
|
||
| std::shared_ptr<TablePropertiesCollectorFactory> | ||
| NewCompactOnDeletionCollectorFactory( | ||
| size_t sliding_window_size, | ||
| size_t deletion_trigger) { | ||
| return std::shared_ptr<TablePropertiesCollectorFactory>( | ||
| new CompactOnDeletionCollectorFactory( | ||
| sliding_window_size, deletion_trigger)); | ||
| } | ||
| } // namespace rocksdb | ||
| #endif // !ROCKSDB_LITE |
101 changes: 101 additions & 0 deletions
101
utilities/table_properties_collectors/compact_on_deletion_collector.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| // Copyright (c) 2015, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| #pragma once | ||
|
|
||
| #ifndef ROCKSDB_LITE | ||
| namespace rocksdb { | ||
|
|
||
| // A factory of a table property collector that marks a SST | ||
| // file as need-compaction when it observe at least "D" deletion | ||
| // entries in any "N" consecutive entires. | ||
| class CompactOnDeletionCollectorFactory | ||
| : public TablePropertiesCollectorFactory { | ||
| public: | ||
| // A factory of a table property collector that marks a SST | ||
| // file as need-compaction when it observe at least "D" deletion | ||
| // entries in any "N" consecutive entires. | ||
| // | ||
| // @param sliding_window_size "N" | ||
| // @param deletion_trigger "D" | ||
| CompactOnDeletionCollectorFactory( | ||
| size_t sliding_window_size, | ||
| size_t deletion_trigger) : | ||
| sliding_window_size_(sliding_window_size), | ||
| deletion_trigger_(deletion_trigger) {} | ||
|
|
||
| virtual ~CompactOnDeletionCollectorFactory() {} | ||
|
|
||
| virtual TablePropertiesCollector* CreateTablePropertiesCollector() override; | ||
|
|
||
| virtual const char* Name() const override { | ||
| return "CompactOnDeletionCollector"; | ||
| } | ||
|
|
||
| private: | ||
| size_t sliding_window_size_; | ||
| size_t deletion_trigger_; | ||
| }; | ||
|
|
||
| class CompactOnDeletionCollector : public TablePropertiesCollector { | ||
| public: | ||
| CompactOnDeletionCollector( | ||
| size_t sliding_window_size, | ||
| size_t deletion_trigger); | ||
|
|
||
| // AddUserKey() will be called when a new key/value pair is inserted into the | ||
| // table. | ||
| // @params key the user key that is inserted into the table. | ||
| // @params value the value that is inserted into the table. | ||
| // @params file_size file size up to now | ||
| virtual Status AddUserKey(const Slice& key, const Slice& value, | ||
| EntryType type, SequenceNumber seq, | ||
| uint64_t file_size) override; | ||
|
|
||
| // Finish() will be called when a table has already been built and is ready | ||
| // for writing the properties block. | ||
| // @params properties User will add their collected statistics to | ||
| // `properties`. | ||
| virtual Status Finish(UserCollectedProperties* properties) override { | ||
| Reset(); | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| // Return the human-readable properties, where the key is property name and | ||
| // the value is the human-readable form of value. | ||
| virtual UserCollectedProperties GetReadableProperties() const override { | ||
| return UserCollectedProperties(); | ||
| } | ||
|
|
||
| // The name of the properties collector can be used for debugging purpose. | ||
| virtual const char* Name() const { | ||
| return "CompactOnDeletionCollector"; | ||
| } | ||
|
|
||
| // EXPERIMENTAL Return whether the output file should be further compacted | ||
| virtual bool NeedCompact() const { | ||
| return need_compaction_; | ||
| } | ||
|
|
||
| static const int kNumBuckets = 128; | ||
|
|
||
| private: | ||
| void Reset(); | ||
|
|
||
| // A ring buffer that used to count the number of deletion entries for every | ||
| // "bucket_size_" keys. | ||
| size_t num_deletions_in_buckets_[kNumBuckets]; | ||
| // the number of keys in a bucket | ||
| size_t bucket_size_; | ||
|
|
||
| size_t current_bucket_; | ||
| size_t num_keys_in_current_bucket_; | ||
| size_t num_deletions_in_observation_window_; | ||
| size_t deletion_trigger_; | ||
| // true if the current SST file needs to be compacted. | ||
| bool need_compaction_; | ||
| }; | ||
| } // namespace rocksdb | ||
| #endif // !ROCKSDB_LITE |
Oops, something went wrong.