Skip to content

Commit

Permalink
build without rocksdb binary dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
azagrebin committed Feb 21, 2019
1 parent fe3fd96 commit e7ab40a
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 97 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ build*/
rocksdbjni-bin
target
java/src/main/resources/lib*
slice.cc
56 changes: 14 additions & 42 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,6 @@ if(NOT ROCKSDBLIBJNI_PATH)
endif()
get_filename_component(ROCKSDB_ABS_PATH ${ROCKSDB_PATH} ABSOLUTE)

if(NOT ROCKSDBLIBJNI_PATH)
set(ROCKSDBLIBJNI_PATH ${CMAKE_SOURCE_DIR}/rocksdbjni-bin)
endif()

if (APPLE)
find_library(rocksdbjni NAMES rocksdbjni-osx PATHS "${ROCKSDBLIBJNI_PATH}")
get_filename_component(ROCKSDBLIBJNI_ABS_PATH ${ROCKSDBLIBJNI_PATH}/librocksdbjni-osx.jnilib ABSOLUTE)
elseif (WIN32)
#SET(CMAKE_FIND_LIBRARY_SUFFIXES ".dll")
#find_library(rocksdbjni NAMES "librocksdbjni-win64" PATHS "${ROCKSDBLIBJNI_PATH}" NO_DEFAULT_PATH)
find_package(RocksDB CONFIG)
set(rocksdbjni RocksDB::rocksdb)
get_filename_component(ROCKSDBLIBJNI_ABS_PATH ${ROCKSDBLIBJNI_PATH}/librocksdbjni-win64.dll ABSOLUTE)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
find_library(rocksdbjni NAMES rocksdbjni-linux-ppc64le PATHS "${ROCKSDBLIBJNI_PATH}")
get_filename_component(ROCKSDBLIBJNI_ABS_PATH ${ROCKSDBLIBJNI_PATH}/librocksdbjni-linux-ppc64le.so ABSOLUTE)
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
find_library(rocksdbjni NAMES rocksdbjni-linux64 PATHS "${ROCKSDBLIBJNI_PATH}")
get_filename_component(ROCKSDBLIBJNI_ABS_PATH ${ROCKSDBLIBJNI_PATH}/librocksdbjni-linux64.so ABSOLUTE)
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "x86")
find_library(rocksdbjni NAMES rocksdbjni-linux32PATHS "${ROCKSDBLIBJNI_PATH}")
get_filename_component(ROCKSDBLIBJNI_ABS_PATH ${ROCKSDBLIBJNI_PATH}/librocksdbjni-linux32.so ABSOLUTE)
else()
message(FATAL_ERROR "Unsupported CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
endif()

file(COPY ${ROCKSDBLIBJNI_ABS_PATH} DESTINATION .)

if(NOT WIN32)
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
endif()
Expand All @@ -64,18 +36,24 @@ include_directories("${ROCKSDB_ABS_PATH}/util")
include_directories("${ROCKSDB_ABS_PATH}/include/rocksdb")
include_directories("${ROCKSDB_ABS_PATH}/include/rocksdb/utilities")

set(SOURCES
src/flink_compaction_filter.cc
src/slice.cc
src/log.cc)

set(JNI_SOURCES
src/flink_compactionfilterjni.cc)

set(TEST_SOURCES
src/flink_compaction_filter_test.cc)

option(WITH_TESTS "build with tests" ON)
if(WITH_TESTS)
include_directories(SYSTEM ${ROCKSDB_ABS_PATH}/third-party/gtest-1.7.0/fused-src)
add_subdirectory(${ROCKSDB_ABS_PATH}/third-party/gtest-1.7.0/fused-src/gtest ${CMAKE_CURRENT_BINARY_DIR}/gtest)

add_executable(flink_compaction_filter_test src/flink_compaction_filter.cc src/flink_compaction_filter_test.cc)
target_link_libraries(flink_compaction_filter_test gtest ${rocksdbjni})
if (APPLE)
add_custom_command(TARGET flink_compaction_filter_test
POST_BUILD COMMAND
${CMAKE_INSTALL_NAME_TOOL} -change "./java/target/librocksdbjni-osx.jnilib" "@loader_path/librocksdbjni-osx.jnilib" $<TARGET_FILE:flink_compaction_filter_test>)
endif()
add_executable(flink_compaction_filter_test ${SOURCES} ${TEST_SOURCES})
target_link_libraries(flink_compaction_filter_test gtest)
endif()

include(FindJava)
Expand All @@ -86,13 +64,7 @@ include_directories(${JNI_INCLUDE_DIRS})
include_directories(${ROCKSDB_ABS_PATH}/java)
include_directories(${ROCKSDB_ABS_PATH}/java/rocksjni)

add_library(rocksdb_plugins SHARED src/flink_compaction_filter.cc src/flink_compactionfilterjni.cc src/flink_compaction_filter.h)
add_library(rocksdb_plugins SHARED ${SOURCES} ${JNI_SOURCES})
if(NOT MSVC)
set_property(TARGET rocksdb_plugins PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
target_link_libraries(rocksdb_plugins ${rocksdbjni})
if (APPLE)
add_custom_command(TARGET rocksdb_plugins
POST_BUILD COMMAND
${CMAKE_INSTALL_NAME_TOOL} -change "./java/target/librocksdbjni-osx.jnilib" "@loader_path/librocksdbjni-osx.jnilib" $<TARGET_FILE:rocksdb_plugins>)
endif()
29 changes: 0 additions & 29 deletions java/src/test/java/org/rocksdb/FlinkCompactionFilterTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@
import org.rocksdb.FlinkCompactionFilter.StateType;
import org.rocksdb.FlinkCompactionFilter.TimeProvider;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
Expand Down Expand Up @@ -152,31 +150,8 @@ private StateContext(StateType type, TimeProvider timeProvider, int timestampOff
cfDesc = new ColumnFamilyDescriptor(getASCII(cf), getOptionsWithFilter(filterFactory));
}

public static void deleteFolder(File folder) {
File[] files = folder.listFiles();
if(files!=null) { //some JVMs return null for empty dirs
for(File f: files) {
if(f.isDirectory()) {
deleteFolder(f);
} else {
f.delete();
}
}
}
}

private Logger createLogger() {
// String path = "/Users/azagrebin/projects/rocksdb_plugins/java";
// try {
// String rdbpath = path + "/java/target";
// deleteFolder(new File(rdbpath));
// NativeLibraryLoader.getInstance().loadLibrary(rdbpath);
// //RocksDB.loadLibrary(Collections.singletonList(path));
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
try (DBOptions opts = new DBOptions().setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL)) {
//System.load(path + "/librocksdb_plugins.dylib");
return new Logger(opts) {
@Override
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
Expand All @@ -197,10 +172,6 @@ private static ColumnFamilyOptions getOptionsWithFilter(
.setMergeOperatorName(MERGE_OPERATOR_NAME);
}

public String getKey() {
return key;
}

ColumnFamilyDescriptor getCfDesc() {
return cfDesc;
}
Expand Down
39 changes: 25 additions & 14 deletions scripts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,27 @@

set -e

CWD=`pwd`

BUILD_DIR=${1:-build}
VERSION=${2:-5.17.2}
ROCKSDB_BIN=${3:-rocksdbjni-bin}
ROCKSDB_PATH=${4:-../rocksdb2}
ROCKSDB_SOURCE_PATH=${3:-../rocksdb2}

unameOut="$(uname -s)"
case "${unameOut}" in
Darwin*) ext=dylib;;
Linux*) ext=so;;
*) echo "UNKNOWN OS : ${unameOut}"; exit 1
esac

LIB=librocksdb_plugins
OUT=${BUILD_DIR}/${LIB}.${ext}
JAVA_OUT=java/src/main/resources/${LIB}.${ext}

#rm -rf ${ROCKSDB_BIN}
scripts/get_rocksdb_dependency.sh ${VERSION} ${ROCKSDB_BIN}
rm -f ${JAVA_OUT}

# copy some rocksdb sources to not depend on rocksdb binaries
cp ${ROCKSDB_SOURCE_PATH}/util/slice.cc src/.

rm -rf ${BUILD_DIR}
mkdir -p ${BUILD_DIR}
Expand All @@ -17,16 +31,13 @@ if hash scl 2>/dev/null; then
scl enable devtoolset-7 'cmake .. -DROCKSDBLIBJNI_PATH=${ROCKSDB_BIN} -DROCKSDB_PATH=${ROCKSDB_PATH}'
scl enable devtoolset-7 'make rocksdb_plugins'
else
cmake .. -DROCKSDBLIBJNI_PATH=${ROCKSDB_BIN} -DROCKSDB_PATH=${ROCKSDB_PATH}
cmake .. -DROCKSDBLIBJNI_PATH=${ROCKSDB_BIN} -DROCKSDB_PATH=${ROCKSDB_SOURCE_PATH}
make rocksdb_plugins
fi
cd ..
cd ${CWD}

LIB=${BUILD_DIR}/librocksdb_plugins
unameOut="$(uname -s)"
case "${unameOut}" in
Darwin*) ext=dylib;;
Linux*) ext=so; patchelf --set-rpath '$ORIGIN' ${LIB}.${ext};;
*) echo "UNKNOWN OS : ${unameOut}"; exit 1
esac
cp ${LIB}.${ext} java/src/main/resources/.
cp ${OUT} ${JAVA_OUT}

cd java
mvn clean install
cd ${CWD}
24 changes: 12 additions & 12 deletions src/flink_compaction_filter_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,16 @@ static const std::size_t LIST_ELEM_FIXED_LEN = static_cast<std::size_t>(8 + 4);

static const int64_t QUERY_TIME_AFTER_NUM_ENTRIES = static_cast<int64_t>(10);

//class ConsoleLogger : public Logger {
//public:
// using Logger::Logv;
// ConsoleLogger() : Logger(InfoLogLevel::DEBUG_LEVEL) {}
//
// void Logv(const char* format, va_list ap) override {
// vprintf(format, ap);
// printf("\n");
// }
//};
class ConsoleLogger : public Logger {
public:
using Logger::Logv;
ConsoleLogger() : Logger(InfoLogLevel::DEBUG_LEVEL) {}

void Logv(const char* format, va_list ap) override {
vprintf(format, ap);
printf("\n");
}
};

int64_t time = 0;

Expand Down Expand Up @@ -89,9 +89,9 @@ void Init(FlinkCompactionFilter::StateType stype,

auto config_holder = std::make_shared<FlinkCompactionFilter::ConfigHolder>();
auto time_provider = new TestTimeProvider();
//auto logger = std::make_shared<ConsoleLogger>();
auto logger = std::make_shared<ConsoleLogger>();

filter = new FlinkCompactionFilter(config_holder, std::unique_ptr<FlinkCompactionFilter::TimeProvider>(time_provider));//, logger);
filter = new FlinkCompactionFilter(config_holder, std::unique_ptr<FlinkCompactionFilter::TimeProvider>(time_provider), logger);
auto config = new FlinkCompactionFilter::Config{state_type, timestamp_offset, ttl, QUERY_TIME_AFTER_NUM_ENTRIES,
unique_ptr<FlinkCompactionFilter::ListElementFilterFactory>(fixed_len_filter_factory)};
EXPECT_EQ(decide(), KKEEP); // test disabled config
Expand Down
83 changes: 83 additions & 0 deletions src/log.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#include "rocksdb/env.h"

#include <thread>
#include "options/db_options.h"
#include "port/port.h"
#include "port/sys_time.h"
#include "rocksdb/options.h"
#include "util/arena.h"
#include "util/autovector.h"

namespace rocksdb {

Logger::~Logger() = default;

Status Logger::Close() {
if (!closed_) {
closed_ = true;
return CloseImpl();
} else {
return Status::OK();
}
}

Status Logger::CloseImpl() { return Status::NotSupported(); }

void Logger::Logv(const InfoLogLevel log_level, const char* format, va_list ap) {
static const char* kInfoLogLevelNames[5] = { "DEBUG", "INFO", "WARN",
"ERROR", "FATAL" };
if (log_level < log_level_) {
return;
}

if (log_level == InfoLogLevel::INFO_LEVEL) {
// Doesn't print log level if it is INFO level.
// This is to avoid unexpected performance regression after we add
// the feature of log level. All the logs before we add the feature
// are INFO level. We don't want to add extra costs to those existing
// logging.
Logv(format, ap);
} else {
char new_format[500];
snprintf(new_format, sizeof(new_format) - 1, "[%s] %s",
kInfoLogLevelNames[log_level], format);
Logv(new_format, ap);
}
}

static void Debugv(Logger* info_log, const char* format, va_list ap) {
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) {
info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap);
}
}

void Debug(Logger* info_log, const char* format, ...) {
va_list ap;
va_start(ap, format);
Debugv(info_log, format, ap);
va_end(ap);
}

static void Errorv(Logger* info_log, const char* format, va_list ap) {
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::ERROR_LEVEL) {
info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap);
}
}

void Error(Logger* info_log, const char* format, ...) {
va_list ap;
va_start(ap, format);
Errorv(info_log, format, ap);
va_end(ap);
}

} // namespace rocksdb

0 comments on commit e7ab40a

Please sign in to comment.