Skip to content

Commit

Permalink
Port folly/synchronization/DistributedMutex to rocksdb (facebook#5642)
Browse files Browse the repository at this point in the history
Summary:
This ports `folly::DistributedMutex` into RocksDB. The PR includes everything else needed to compile and use DistributedMutex as a component within folly. Most files are unchanged except for some portability stuff and includes.

For now, I've put this under `rocksdb/third-party`, but if there is a better folder to put this under, let me know. I also am not sure how or where to put unit tests for third-party stuff like this. It seems like gtest is included already, but I need to link with it from another third-party folder.

This also includes some other common components from folly

- folly/Optional
- folly/ScopeGuard (In particular `SCOPE_EXIT`)
- folly/synchronization/ParkingLot (A portable futex-like interface)
- folly/synchronization/AtomicNotification (The standard C++ interface for futexes)
- folly/Indestructible (For singletons that don't get destroyed without allocations)
Pull Request resolved: facebook#5642

Differential Revision: D16544439

fbshipit-source-id: 179b98b5dcddc3075926d31a30f92fd064245731
  • Loading branch information
aary authored and facebook-github-bot committed Aug 7, 2019
1 parent 6e78fe3 commit 38b03c8
Show file tree
Hide file tree
Showing 48 changed files with 7,335 additions and 1 deletion.
23 changes: 23 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files,
if (WITH_WINDOWS_UTF8_FILENAMES)
add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
endif()
# third-party/folly is only validated to work on Linux and Windows for now.
# So only turn it on there by default.
if(CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows")
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" ON)
else()
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
endif()
if(MSVC)
# Defaults currently different for GFLAGS.
# We will address find_package work a little later
Expand Down Expand Up @@ -462,6 +469,9 @@ endif()
include_directories(${PROJECT_SOURCE_DIR})
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
endif()
find_package(Threads REQUIRED)

# Main library source code
Expand Down Expand Up @@ -738,6 +748,15 @@ else()
env/io_posix.cc)
endif()

if(WITH_FOLLY_DISTRIBUTED_MUTEX)
list(APPEND SOURCES
third-party/folly/folly/detail/Futex.cpp
third-party/folly/folly/synchronization/AtomicNotification.cpp
third-party/folly/folly/synchronization/DistributedMutex.cpp
third-party/folly/folly/synchronization/ParkingLot.cpp
third-party/folly/folly/synchronization/WaitOptions.cpp)
endif()

set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
Expand Down Expand Up @@ -1009,6 +1028,10 @@ if(WITH_TESTS)
list(APPEND TESTS utilities/env_librados_test.cc)
endif()

if(WITH_FOLLY_DISTRIBUTED_MUTEX)
list(APPEND TESTS third-party/folly/folly/synchronization/test/DistributedMutexTest.cpp)
endif()

set(BENCHMARKS
cache/cache_bench.cc
memtable/memtablerep_bench.cc
Expand Down
30 changes: 29 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ endif

ifeq ($(MAKECMDGOALS),rocksdbjavastaticreleasedocker)
ifneq ($(DEBUG_LEVEL),2)
DEBUG_LEVEL=0
DEBUG_LEVEL=0
endif
endif

Expand Down Expand Up @@ -304,6 +304,10 @@ ifndef DISABLE_JEMALLOC
PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
endif

ifndef USE_FOLLY_DISTRIBUTED_MUTEX
USE_FOLLY_DISTRIBUTED_MUTEX=0
endif

export GTEST_THROW_ON_FAILURE=1
export GTEST_HAS_EXCEPTIONS=1
GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
Expand All @@ -316,6 +320,18 @@ else
PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
endif

ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
FOLLY_DIR = ./third-party/folly
# AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(FOLLY_DIR)
PLATFORM_CXXFLAGS += -I$(FOLLY_DIR)
else
PLATFORM_CCFLAGS += -isystem $(FOLLY_DIR)
PLATFORM_CXXFLAGS += -isystem $(FOLLY_DIR)
endif
endif

# This (the first rule) must depend on "all".
default: all

Expand Down Expand Up @@ -402,6 +418,9 @@ endif

LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
MOCKOBJECTS = $(MOCK_LIB_SOURCES:.cc=.o)
ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
FOLLYOBJECTS = $(FOLLY_SOURCES:.cpp=.o)
endif

GTEST = $(GTEST_DIR)/gtest/gtest-all.o
TESTUTIL = ./test_util/testutil.o
Expand Down Expand Up @@ -569,6 +588,10 @@ TESTS = \
block_cache_tracer_test \
block_cache_trace_analyzer_test \

ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
TESTS += folly_synchronization_distributed_mutex_test
endif

PARALLEL_TEST = \
backupable_db_test \
db_bloom_filter_test \
Expand Down Expand Up @@ -1120,6 +1143,11 @@ trace_analyzer: tools/trace_analyzer.o $(ANALYZETOOLOBJECTS) $(LIBOBJECTS)
block_cache_trace_analyzer: tools/block_cache_analyzer/block_cache_trace_analyzer_tool.o $(ANALYZETOOLOBJECTS) $(LIBOBJECTS)
$(AM_LINK)

ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
folly_synchronization_distributed_mutex_test: $(LIBOBJECTS) $(TESTHARNESS) $(FOLLYOBJECTS) third-party/folly/folly/synchronization/test/DistributedMutexTest.o
$(AM_LINK)
endif

cache_bench: cache/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
$(AM_LINK)

Expand Down
6 changes: 6 additions & 0 deletions build_tools/build_detect_platform
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ case "$TARGET_OS" in
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
fi
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
if test -z "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
USE_FOLLY_DISTRIBUTED_MUTEX=1
fi
# PORT_FILES=port/linux/linux_specific.cc
;;
SunOS)
Expand Down Expand Up @@ -661,3 +664,6 @@ if test -n "$WITH_JEMALLOC_FLAG"; then
echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
fi
echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
if test -n "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
echo "USE_FOLLY_DISTRIBUTED_MUTEX=$USE_FOLLY_DISTRIBUTED_MUTEX" >> "$OUTPUT"
fi
2 changes: 2 additions & 0 deletions build_tools/fbcode_config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -159,4 +159,6 @@ else
LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
fi

USE_FOLLY_DISTRIBUTED_MUTEX=1

export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
2 changes: 2 additions & 0 deletions build_tools/fbcode_config_platform007.sh
Original file line number Diff line number Diff line change
Expand Up @@ -155,4 +155,6 @@ VALGRIND_VER="$VALGRIND_BASE/bin/"
LUA_PATH=
LUA_LIB=

USE_FOLLY_DISTRIBUTED_MUTEX=1

export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
7 changes: 7 additions & 0 deletions src.mk
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,13 @@ TEST_LIB_SOURCES = \
test_util/testutil.cc \
utilities/cassandra/test_utils.cc \

FOLLY_SOURCES = \
third-party/folly/folly/detail/Futex.cpp \
third-party/folly/folly/synchronization/AtomicNotification.cpp \
third-party/folly/folly/synchronization/DistributedMutex.cpp \
third-party/folly/folly/synchronization/ParkingLot.cpp \
third-party/folly/folly/synchronization/WaitOptions.cpp \

MAIN_SOURCES = \
cache/cache_bench.cc \
cache/cache_test.cc \
Expand Down
15 changes: 15 additions & 0 deletions third-party/folly/folly/CPortability.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).

#pragma once

/**
* Macro for marking functions as having public visibility.
*/
#if defined(__GNUC__)
#define FOLLY_EXPORT __attribute__((__visibility__("default")))
#else
#define FOLLY_EXPORT
#endif
17 changes: 17 additions & 0 deletions third-party/folly/folly/ConstexprMath.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).

#pragma once

namespace folly {
template <typename T>
constexpr T constexpr_max(T a) {
return a;
}
template <typename T, typename... Ts>
constexpr T constexpr_max(T a, T b, Ts... ts) {
return b < a ? constexpr_max(a, ts...) : constexpr_max(b, ts...);
}
} // namespace folly
166 changes: 166 additions & 0 deletions third-party/folly/folly/Indestructible.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).

#pragma once

#include <cassert>
#include <type_traits>
#include <utility>

#include <folly/Traits.h>

namespace folly {

/***
* Indestructible
*
* When you need a Meyers singleton that will not get destructed, even at
* shutdown, and you also want the object stored inline.
*
* Use like:
*
* void doSomethingWithExpensiveData();
*
* void doSomethingWithExpensiveData() {
* static const Indestructible<map<string, int>> data{
* map<string, int>{{"key1", 17}, {"key2", 19}, {"key3", 23}},
* };
* callSomethingTakingAMapByRef(*data);
* }
*
* This should be used only for Meyers singletons, and, even then, only when
* the instance does not need to be destructed ever.
*
* This should not be used more generally, e.g., as member fields, etc.
*
* This is designed as an alternative, but with one fewer allocation at
* construction time and one fewer pointer dereference at access time, to the
* Meyers singleton pattern of:
*
* void doSomethingWithExpensiveData() {
* static const auto data = // never `delete`d
* new map<string, int>{{"key1", 17}, {"key2", 19}, {"key3", 23}};
* callSomethingTakingAMapByRef(*data);
* }
*/

template <typename T>
class Indestructible final {
public:
template <typename S = T, typename = decltype(S())>
constexpr Indestructible() noexcept(noexcept(T())) {}

/**
* Constructor accepting a single argument by forwarding reference, this
* allows using list initialzation without the overhead of things like
* in_place, etc and also works with std::initializer_list constructors
* which can't be deduced, the default parameter helps there.
*
* auto i = folly::Indestructible<std::map<int, int>>{{{1, 2}}};
*
* This provides convenience
*
* There are two versions of this constructor - one for when the element is
* implicitly constructible from the given argument and one for when the
* type is explicitly but not implicitly constructible from the given
* argument.
*/
template <
typename U = T,
_t<std::enable_if<std::is_constructible<T, U&&>::value>>* = nullptr,
_t<std::enable_if<
!std::is_same<Indestructible<T>, remove_cvref_t<U>>::value>>* =
nullptr,
_t<std::enable_if<!std::is_convertible<U&&, T>::value>>* = nullptr>
explicit constexpr Indestructible(U&& u) noexcept(
noexcept(T(std::declval<U>())))
: storage_(std::forward<U>(u)) {}
template <
typename U = T,
_t<std::enable_if<std::is_constructible<T, U&&>::value>>* = nullptr,
_t<std::enable_if<
!std::is_same<Indestructible<T>, remove_cvref_t<U>>::value>>* =
nullptr,
_t<std::enable_if<std::is_convertible<U&&, T>::value>>* = nullptr>
/* implicit */ constexpr Indestructible(U&& u) noexcept(
noexcept(T(std::declval<U>())))
: storage_(std::forward<U>(u)) {}

template <typename... Args, typename = decltype(T(std::declval<Args>()...))>
explicit constexpr Indestructible(Args&&... args) noexcept(
noexcept(T(std::declval<Args>()...)))
: storage_(std::forward<Args>(args)...) {}
template <
typename U,
typename... Args,
typename = decltype(
T(std::declval<std::initializer_list<U>&>(),
std::declval<Args>()...))>
explicit constexpr Indestructible(std::initializer_list<U> il, Args... args) noexcept(
noexcept(
T(std::declval<std::initializer_list<U>&>(),
std::declval<Args>()...)))
: storage_(il, std::forward<Args>(args)...) {}

~Indestructible() = default;

Indestructible(Indestructible const&) = delete;
Indestructible& operator=(Indestructible const&) = delete;

Indestructible(Indestructible&& other) noexcept(
noexcept(T(std::declval<T>())))
: storage_(std::move(other.storage_.value)) {
other.erased_ = true;
}
Indestructible& operator=(Indestructible&& other) noexcept(
noexcept(T(std::declval<T>()))) {
storage_.value = std::move(other.storage_.value);
other.erased_ = true;
}

T* get() noexcept {
check();
return &storage_.value;
}
T const* get() const noexcept {
check();
return &storage_.value;
}
T& operator*() noexcept {
return *get();
}
T const& operator*() const noexcept {
return *get();
}
T* operator->() noexcept {
return get();
}
T const* operator->() const noexcept {
return get();
}

private:
void check() const noexcept {
assert(!erased_);
}

union Storage {
T value;

template <typename S = T, typename = decltype(S())>
constexpr Storage() noexcept(noexcept(T())) : value() {}

template <typename... Args, typename = decltype(T(std::declval<Args>()...))>
explicit constexpr Storage(Args&&... args) noexcept(
noexcept(T(std::declval<Args>()...)))
: value(std::forward<Args>(args)...) {}

~Storage() {}
};

Storage storage_{};
bool erased_{false};
};
} // namespace folly
Loading

0 comments on commit 38b03c8

Please sign in to comment.