forked from facebook/rocksdb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Cassandra compaction filter for purge expired columns and rows
Summary: Major changes in this PR: * Implement CassandraCompactionFilter to remove expired columns and rows (if all column expired) * Move cassandra related code from utilities/merge_operators/cassandra to utilities/cassandra/* * Switch to use shared_ptr<> from uniqu_ptr for Column membership management in RowValue. Since columns do have multiple owners in Merge and GC process, use shared_ptr helps make RowValue immutable. * Rename cassandra_merge_test to cassandra_functional_test and add two TTL compaction related tests there. Closes facebook#2588 Differential Revision: D5430010 Pulled By: wpc fbshipit-source-id: 9566c21e06de17491d486a68c70f52d501f27687
- Loading branch information
1 parent
63163a8
commit 534c255
Showing
23 changed files
with
617 additions
and
240 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under both the GPLv2 (found in the | ||
| // COPYING file in the root directory) and Apache 2.0 License | ||
| // (found in the LICENSE.Apache file in the root directory). | ||
|
|
||
| #include <jni.h> | ||
|
|
||
| #include "include/org_rocksdb_CassandraCompactionFilter.h" | ||
| #include "utilities/cassandra/cassandra_compaction_filter.h" | ||
|
|
||
| /* | ||
| * Class: org_rocksdb_CassandraCompactionFilter | ||
| * Method: createNewCassandraCompactionFilter0 | ||
| * Signature: ()J | ||
| */ | ||
| jlong Java_org_rocksdb_CassandraCompactionFilter_createNewCassandraCompactionFilter0( | ||
| JNIEnv* env, jclass jcls, jboolean purge_ttl_on_expiration) { | ||
| auto* compaction_filter = | ||
| new rocksdb::cassandra::CassandraCompactionFilter(purge_ttl_on_expiration); | ||
| // set the native handle to our native compaction filter | ||
| return reinterpret_cast<jlong>(compaction_filter); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
java/src/main/java/org/rocksdb/CassandraCompactionFilter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. An additional grant | ||
| // of patent rights can be found in the PATENTS file in the same directory. | ||
|
|
||
| package org.rocksdb; | ||
|
|
||
| /** | ||
| * Just a Java wrapper around CassandraCompactionFilter implemented in C++ | ||
| */ | ||
| public class CassandraCompactionFilter | ||
| extends AbstractCompactionFilter<Slice> { | ||
| public CassandraCompactionFilter(boolean purgeTtlOnExpiration) { | ||
| super(createNewCassandraCompactionFilter0(purgeTtlOnExpiration)); | ||
| } | ||
|
|
||
| private native static long createNewCassandraCompactionFilter0(boolean purgeTtlOnExpiration); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under both the GPLv2 (found in the | ||
| // COPYING file in the root directory) and Apache 2.0 License | ||
| // (found in the LICENSE.Apache file in the root directory). | ||
|
|
||
| #include "utilities/cassandra/cassandra_compaction_filter.h" | ||
| #include <string> | ||
| #include "rocksdb/slice.h" | ||
| #include "utilities/cassandra/format.h" | ||
|
|
||
|
|
||
| namespace rocksdb { | ||
| namespace cassandra { | ||
|
|
||
| const char* CassandraCompactionFilter::Name() const { | ||
| return "CassandraCompactionFilter"; | ||
| } | ||
|
|
||
| CompactionFilter::Decision CassandraCompactionFilter::FilterV2( | ||
| int level, | ||
| const Slice& key, | ||
| ValueType value_type, | ||
| const Slice& existing_value, | ||
| std::string* new_value, | ||
| std::string* skip_until) const { | ||
|
|
||
| bool value_changed = false; | ||
| RowValue row_value = RowValue::Deserialize( | ||
| existing_value.data(), existing_value.size()); | ||
| RowValue compacted = purge_ttl_on_expiration_ ? | ||
| row_value.PurgeTtl(&value_changed) : | ||
| row_value.ExpireTtl(&value_changed); | ||
|
|
||
| if(compacted.Empty()) { | ||
| return Decision::kRemove; | ||
| } | ||
|
|
||
| if (value_changed) { | ||
| compacted.Serialize(new_value); | ||
| return Decision::kChangeValue; | ||
| } | ||
|
|
||
| return Decision::kKeep; | ||
| } | ||
|
|
||
| } // namespace cassandra | ||
| } // namespace rocksdb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. | ||
| // This source code is licensed under both the GPLv2 (found in the | ||
| // COPYING file in the root directory) and Apache 2.0 License | ||
| // (found in the LICENSE.Apache file in the root directory). | ||
|
|
||
| #pragma once | ||
| #include <string> | ||
| #include "rocksdb/compaction_filter.h" | ||
| #include "rocksdb/slice.h" | ||
|
|
||
| namespace rocksdb { | ||
| namespace cassandra { | ||
|
|
||
| /** | ||
| * Compaction filter for removing expired Cassandra data with ttl. | ||
| * If option `purge_ttl_on_expiration` is set to true, expired data | ||
| * will be directly purged. Otherwise expired data will be converted | ||
| * tombstones first, then be eventally removed after gc grace period. | ||
| * `purge_ttl_on_expiration` should only be on in the case all the | ||
| * writes have same ttl setting, otherwise it could bring old data back. | ||
| */ | ||
| class CassandraCompactionFilter : public CompactionFilter { | ||
| public: | ||
| explicit CassandraCompactionFilter(bool purge_ttl_on_expiration) | ||
| : purge_ttl_on_expiration_(purge_ttl_on_expiration) {} | ||
|
|
||
| const char* Name() const override; | ||
| virtual Decision FilterV2(int level, | ||
| const Slice& key, | ||
| ValueType value_type, | ||
| const Slice& existing_value, | ||
| std::string* new_value, | ||
| std::string* skip_until) const override; | ||
|
|
||
| private: | ||
| bool purge_ttl_on_expiration_; | ||
| }; | ||
| } // namespace cassandra | ||
| } // namespace rocksdb |
Oops, something went wrong.