Skip to content

Commit

Permalink
Introduce Lua Extension: RocksLuaCompactionFilter
Browse files Browse the repository at this point in the history
Summary:
This diff includes an implementation of CompactionFilter that allows
users to write CompactionFilter in Lua.  With this ability, users can
dynamically change compaction filter logic without requiring building
the rocksdb binary and restarting the database.

To compile, WITH_LUA_PATH must be specified to the base directory
of lua.
Closes facebook#1478

Differential Revision: D4150138

Pulled By: yhchiang

fbshipit-source-id: ed84222
  • Loading branch information
yhchiang authored and Facebook Github Bot committed Nov 16, 2016
1 parent 760ef68 commit 647eafd
Show file tree
Hide file tree
Showing 14 changed files with 1,077 additions and 2 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,7 @@ set(SOURCES
utilities/flashcache/flashcache.cc
utilities/geodb/geodb_impl.cc
utilities/leveldb_options/leveldb_options.cc
utilities/lua/rocks_lua_compaction_filter.cc
utilities/memory/memory_util.cc
utilities/merge_operators/string_append/stringappend.cc
utilities/merge_operators/string_append/stringappend2.cc
Expand Down Expand Up @@ -601,6 +602,7 @@ set(TESTS
utilities/ttl/ttl_test.cc
utilities/write_batch_with_index/write_batch_with_index_test.cc
utilities/column_aware_encoding_test.cc
utilities/lua/rocks_lua_test.cc
)
if(WITH_LIBRADOS)
list(APPEND TESTS utilities/env_librados_test.cc)
Expand Down
1 change: 1 addition & 0 deletions HISTORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
### New Features
* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions().
* Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details.
* Add LuaCompactionFilter in utilities. This allows developers to write compaction filters in Lua. To use this feature, LUA_PATH needs to be set to the root directory of Lua.

## 4.13.0 (10/18/2016)
### Public API Change
Expand Down
35 changes: 35 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ ifneq ($(filter -DROCKSDB_LITE,$(OPT)),)
# found
CFLAGS += -fno-exceptions
CXXFLAGS += -fno-exceptions
# LUA is not supported under ROCKSDB_LITE
LUA_PATH =
endif

# ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
Expand All @@ -185,6 +187,8 @@ ifdef COMPILE_WITH_TSAN
# Turn off -pg when enabling TSAN testing, because that induces
# a link failure. TODO: find the root cause
PROFILING_FLAGS =
# LUA is not supported under TSAN
LUA_PATH =
endif

# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
Expand Down Expand Up @@ -220,6 +224,34 @@ ifndef DISABLE_WARNING_AS_ERROR
WARNING_FLAGS += -Werror
endif


ifdef LUA_PATH

ifndef LUA_INCLUDE
LUA_INCLUDE=$(LUA_PATH)/include
endif

LUA_INCLUDE_FILE=$(LUA_INCLUDE)/lualib.h

ifeq ("$(wildcard $(LUA_INCLUDE_FILE))", "")
# LUA_INCLUDE_FILE does not exist
$(error Cannot find lualib.h under $(LUA_INCLUDE). Try to specify both LUA_PATH and LUA_INCLUDE manually)
endif
LUA_FLAGS = -I$(LUA_INCLUDE) -DLUA -DLUA_COMPAT_ALL
CFLAGS += $(LUA_FLAGS)
CXXFLAGS += $(LUA_FLAGS)

ifndef LUA_LIB
LUA_LIB = $(LUA_PATH)/lib/liblua.a
endif
ifeq ("$(wildcard $(LUA_LIB))", "") # LUA_LIB does not exist
$(error $(LUA_LIB) does not exist. Try to specify both LUA_PATH and LUA_LIB manually)
endif
LDFLAGS += $(LUA_LIB)

endif


CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers

Expand Down Expand Up @@ -380,6 +412,7 @@ TESTS = \
iostats_context_test \
persistent_cache_test \
statistics_test \
lua_test \
lru_cache_test \

PARALLEL_TEST = \
Expand Down Expand Up @@ -1260,6 +1293,8 @@ statistics_test: util/statistics_test.o $(LIBOBJECTS) $(TESTHARNESS)
lru_cache_test: util/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)

lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)

#-------------------------------------------------
# make install related stuff
Expand Down
1 change: 1 addition & 0 deletions build_tools/build_detect_platform
Original file line number Diff line number Diff line change
Expand Up @@ -466,3 +466,4 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
if test -n "$JEMALLOC"; then
echo "JEMALLOC=1" >> "$OUTPUT"
fi
echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
1 change: 1 addition & 0 deletions build_tools/dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@ TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.10.0/gcc-4.9-glibc-2.20/e9936bf
LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-4.9-glibc-2.20/690f0d7
1 change: 1 addition & 0 deletions build_tools/dependencies_4.8.1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@ TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e
10 changes: 9 additions & 1 deletion build_tools/fbcode_config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,12 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GF

VALGRIND_VER="$VALGRIND_BASE/bin/"

export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
LUA_PATH="$LUA_BASE"

if test -z $PIC_BUILD; then
LUA_LIB=" $LUA_PATH/lib/liblua.a"
else
LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
fi

export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
4 changes: 3 additions & 1 deletion build_tools/fbcode_config4.8.1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -110,4 +110,6 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $G

VALGRIND_VER="$VALGRIND_BASE/bin/"

export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
LUA_PATH="$LUA_BASE"

export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH
2 changes: 2 additions & 0 deletions build_tools/update_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ get_lib_base tbb 4.0_update2 gcc-4.9-glibc-2.20
get_lib_base kernel-headers LATEST
get_lib_base binutils LATEST centos6-native
get_lib_base valgrind 3.10.0 gcc-4.9-glibc-2.20
get_lib_base lua 5.2.3 gcc-4.9-glibc-2.20

git diff $OUTPUT

Expand Down Expand Up @@ -125,5 +126,6 @@ get_lib_base tbb 4.0_update2 gcc-4.8.1-glibc-2.17
get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17
get_lib_base binutils LATEST centos6-native
get_lib_base valgrind 3.8.1 gcc-4.8.1-glibc-2.17
get_lib_base lua 5.2.3 centos6-native

git diff $OUTPUT
187 changes: 187 additions & 0 deletions include/rocksdb/utilities/lua/rocks_lua_compaction_filter.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.

#pragma once

#if defined(LUA) && !defined(ROCKSDB_LITE)
// lua headers
extern "C" {
#include <lauxlib.h>
#include <lua.h>
#include <lualib.h>
}

#include <mutex>
#include <string>
#include <vector>

#include "rocksdb/compaction_filter.h"
#include "rocksdb/env.h"
#include "rocksdb/slice.h"
#include "rocksdb/utilities/lua/rocks_lua_custom_library.h"
#include "rocksdb/utilities/lua/rocks_lua_util.h"

namespace rocksdb {
namespace lua {

struct RocksLuaCompactionFilterOptions {
// The lua script in string that implements all necessary CompactionFilter
// virtual functions. The specified lua_script must implement the following
// functions, which are Name and Filter, as described below.
//
// 0. The Name function simply returns a string representing the name of
// the lua script. If there's any erorr in the Name function, an
// empty string will be used.
// --- Example
// function Name()
// return "DefaultLuaCompactionFilter"
// end
//
//
// 1. The script must contains a function called Filter, which implements
// CompactionFilter::Filter() , takes three input arguments, and returns
// three values as the following API:
//
// function Filter(level, key, existing_value)
// ...
// return is_filtered, is_changed, new_value
// end
//
// Note that if ignore_value is set to true, then Filter should implement
// the following API:
//
// function Filter(level, key)
// ...
// return is_filtered
// end
//
// If there're any error in the Filter() function, then it will keep
// the input key / value pair.
//
// -- Input
// The function must take three arguments (integer, string, string),
// which map to "level", "key", and "existing_value" passed from
// RocksDB.
//
// -- Output
// The function must return three values (boolean, boolean, string).
// - is_filtered: if the first return value is true, then it indicates
// the input key / value pair should be filtered.
// - is_changed: if the second return value is true, then it indicates
// the existing_value needs to be changed, and the resulting value
// is stored in the third return value.
// - new_value: if the second return value is true, then this third
// return value stores the new value of the input key / value pair.
//
// -- Examples
// -- a filter that keeps all key-value pairs
// function Filter(level, key, existing_value)
// return false, false, ""
// end
//
// -- a filter that keeps all keys and change their values to "Rocks"
// function Filter(level, key, existing_value)
// return false, true, "Rocks"
// end

std::string lua_script;

// If set to true, then existing_value will not be passed to the Filter
// function, and the Filter function only needs to return a single boolean
// flag indicating whether to filter out this key or not.
//
// function Filter(level, key)
// ...
// return is_filtered
// end
bool ignore_value = false;

// A boolean flag to determine whether to ignore snapshots.
bool ignore_snapshots = false;

// When specified a non-null pointer, the first "error_limit_per_filter"
// errors of each CompactionFilter that is lua related will be included
// in this log.
std::shared_ptr<Logger> error_log;

// The number of errors per CompactionFilter will be printed
// to error_log.
int error_limit_per_filter = 1;

// A string to luaL_reg array map that allows the Lua CompactionFilter
// to use custom C library. The string will be used as the library
// name in Lua.
std::vector<std::shared_ptr<RocksLuaCustomLibrary>> libraries;

///////////////////////////////////////////////////////////////////////////
// NOT YET SUPPORTED
// The name of the Lua function in "lua_script" that implements
// CompactionFilter::FilterMergeOperand(). The function must take
// three input arguments (integer, string, string), which map to "level",
// "key", and "operand" passed from the RocksDB. In addition, the
// function must return a single boolean value, indicating whether
// to filter the input key / operand.
//
// DEFAULT: the default implementation always returns false.
// @see CompactionFilter::FilterMergeOperand
};

class RocksLuaCompactionFilterFactory : public CompactionFilterFactory {
public:
explicit RocksLuaCompactionFilterFactory(
const RocksLuaCompactionFilterOptions opt);

virtual ~RocksLuaCompactionFilterFactory() {}

std::unique_ptr<CompactionFilter> CreateCompactionFilter(
const CompactionFilter::Context& context) override;

// Change the Lua script so that the next compaction after this
// function call will use the new Lua script.
void SetScript(const std::string& new_script);

// Obtain the current Lua script
std::string GetScript();

const char* Name() const override;

private:
RocksLuaCompactionFilterOptions opt_;
std::string name_;
// A lock to protect "opt_" to make it dynamically changeable.
std::mutex opt_mutex_;
};

// A wrapper class that invokes Lua script to perform CompactionFilter
// functions.
class RocksLuaCompactionFilter : public rocksdb::CompactionFilter {
public:
explicit RocksLuaCompactionFilter(const RocksLuaCompactionFilterOptions& opt)
: options_(opt),
lua_state_wrapper_(opt.lua_script, opt.libraries),
error_count_(0) {}

virtual bool Filter(int level, const Slice& key, const Slice& existing_value,
std::string* new_value,
bool* value_changed) const override;
// Not yet supported
virtual bool FilterMergeOperand(int level, const Slice& key,
const Slice& operand) const override {
return false;
}
virtual bool IgnoreSnapshots() const override;
virtual const char* Name() const override;

protected:
void LogLuaError(const char* format, ...) const;

RocksLuaCompactionFilterOptions options_;
LuaStateWrapper lua_state_wrapper_;
mutable int error_count_;
};

} // namespace lua
} // namespace rocksdb
#endif // defined(LUA) && !defined(ROCKSDB_LITE)
43 changes: 43 additions & 0 deletions include/rocksdb/utilities/lua/rocks_lua_custom_library.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.

#pragma once
#ifdef LUA

// lua headers
extern "C" {
#include <lauxlib.h>
#include <lua.h>
#include <lualib.h>
}

namespace rocksdb {
namespace lua {
// A class that used to define custom C Library that is callable
// from Lua script
class RocksLuaCustomLibrary {
public:
virtual ~RocksLuaCustomLibrary() {}
// The name of the C library. This name will also be used as the table
// (namespace) in Lua that contains the C library.
virtual const char* Name() const = 0;

// Returns a "static const struct luaL_Reg[]", which includes a list of
// C functions. Note that the last entry of this static array must be
// {nullptr, nullptr} as required by Lua.
//
// More details about how to implement Lua C libraries can be found
// in the official Lua document http://www.lua.org/pil/26.2.html
virtual const struct luaL_Reg* Lib() const = 0;

// A function that will be called right after the library has been created
// and pushed on the top of the lua_State. This custom setup function
// allows developers to put additional table or constant values inside
// the same table / namespace.
virtual void CustomSetup(lua_State* L) const {}
};
} // namespace lua
} // namespace rocksdb
#endif // LUA
Loading

0 comments on commit 647eafd

Please sign in to comment.