merge operator with delim

azagrebin · Feb 21, 2019 · 65b2a4c · 65b2a4c
1 parent e3d02e1
commit 65b2a4c
Show file tree

Hide file tree

Showing 19 changed files with 617 additions and 170 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,12 @@
 .idea
 *.iml
+
 cmake-build*/
 build*/
-rocksdbjni-bin
+
 target
 java/src/main/resources/lib*
-slice.cc
+
+/rocksdb/
+slice.cc
+merge_operator.cc
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,18 @@
 cmake_minimum_required(VERSION 2.8)
 
-project(rocksdb_plugins)
+if(NOT LIBNAME)
+    set(LIBNAME frocksdbplugins)
+endif()
+
+if(NOT ROCKSDB_SOURCE_PATH)
+    set(ROCKSDB_SOURCE_PATH ${CMAKE_SOURCE_DIR}/rocksdb)
+endif()
+
+if (NOT TESTS)
+    set(TESTS flink_compaction_filter_test)
+endif()
+
+project(${LIBNAME})
 enable_language(CXX)
 enable_language(C)
 
@@ -11,10 +23,7 @@ endif()
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 
-if(NOT ROCKSDBLIBJNI_PATH)
-    set(ROCKSDB_PATH ${CMAKE_SOURCE_DIR}/../rocksdb)
-endif()
-get_filename_component(ROCKSDB_ABS_PATH ${ROCKSDB_PATH} ABSOLUTE)
+get_filename_component(ROCKSDB_ABS_PATH ${ROCKSDB_SOURCE_PATH} ABSOLUTE)
 
 if(NOT WIN32)
     add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
@@ -35,36 +44,44 @@ include_directories("${ROCKSDB_ABS_PATH}/include")
 include_directories("${ROCKSDB_ABS_PATH}/util")
 include_directories("${ROCKSDB_ABS_PATH}/include/rocksdb")
 include_directories("${ROCKSDB_ABS_PATH}/include/rocksdb/utilities")
+include_directories("${ROCKSDB_ABS_PATH}/java")
+include_directories("${ROCKSDB_ABS_PATH}/java/rocksjni")
 
 set(SOURCES
         src/flink_compaction_filter.cc
+        src/stringappend_delim.cc
+        src/log.cc
         src/slice.cc
-        src/log.cc)
+        src/merge_operator.cc)
 
 set(JNI_SOURCES
-        src/flink_compactionfilterjni.cc)
+        src/flink_compactionfilterjni.cc
+        src/stringappend_delim_jni.cc)
+
+include(FindJava)
+include(UseJava)
+include(FindJNI)
+include_directories("${JNI_INCLUDE_DIRS}")
 
-set(TEST_SOURCES
-        src/flink_compaction_filter_test.cc)
+add_library(${LIBNAME} SHARED ${SOURCES} ${JNI_SOURCES})
+
+# tests
 
 option(WITH_TESTS "build with tests" ON)
 if(WITH_TESTS)
-    include_directories(SYSTEM ${ROCKSDB_ABS_PATH}/third-party/gtest-1.7.0/fused-src)
-    add_subdirectory(${ROCKSDB_ABS_PATH}/third-party/gtest-1.7.0/fused-src/gtest ${CMAKE_CURRENT_BINARY_DIR}/gtest)
-
-    add_executable(flink_compaction_filter_test ${SOURCES} ${TEST_SOURCES})
-    target_link_libraries(flink_compaction_filter_test gtest)
-endif()
+    enable_testing()
 
-include(FindJava)
-include(UseJava)
-include(FindJNI)
+    foreach(TEST ${TESTS})
+        list(APPEND TEST_SOURCES "src/${TEST}.cc")
+    endforeach()
 
-include_directories(${JNI_INCLUDE_DIRS})
-include_directories(${ROCKSDB_ABS_PATH}/java)
-include_directories(${ROCKSDB_ABS_PATH}/java/rocksjni)
+    include_directories(SYSTEM ${ROCKSDB_ABS_PATH}/third-party/gtest-1.7.0/fused-src)
+    add_subdirectory(${ROCKSDB_ABS_PATH}/third-party/gtest-1.7.0/fused-src/gtest ${CMAKE_CURRENT_BINARY_DIR}/gtest)
 
-add_library(rocksdb_plugins SHARED ${SOURCES} ${JNI_SOURCES})
-if(NOT MSVC)
-    set_property(TARGET rocksdb_plugins PROPERTY POSITION_INDEPENDENT_CODE ON)
+    foreach(TEST ${TESTS})
+        add_executable(${TEST} ${SOURCES} ${TEST_SOURCES})
+        target_link_libraries(${TEST} gtest pthread)
+        add_test(${TEST} ${TEST})
+        add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} DEPENDS ${TEST})
+    endforeach()
 endif()
diff --git a/Makefile b/Makefile
@@ -0,0 +1,34 @@
+
+ROCKSDB_VERSION ?= 5.17.2
+
+.PHONY: $(MAKECMDGOALS)
+
+# Crossbuild:
+
+all: clean build-linux32 build-linux64 build-osx
+
+prepare-rocksdb:
+	git clone https://github.com/facebook/rocksdb
+	cd rocksdb; git checkout v${ROCKSDB_VERSION}
+
+clean:
+	# docker rm -f rocksdb_linux_x86-be || true
+	# docker rm -f rocksdb_linux_x64-be || true
+	rm -rf build-*
+
+build-linux32:
+	./scripts/build-linux-docker.sh 32
+
+build-linux64:
+	./scripts/build-linux-docker.sh 64
+
+build-osx:
+	./scripts/build.sh osx
+
+# Only for windows:
+
+win-prepare: prepare-rocksdb
+	choco install jdk8 maven visualstudio2017community intellijidea-community vscode
+
+build-win:
+	.\scripts\build-win.bat
diff --git a/README.md b/README.md
@@ -0,0 +1,51 @@
+
+## Build in PPC64LE
+
+ Use Ubuntu 16.04 (e.g. AWS instance 4 cores, 16GB RAM, 40GB storage for build).
+ Install git if not installed. If docker is installed, it might need to be removed.
+
+ Setup ppc64le docker machine ([source](https://developer.ibm.com/linuxonpower/2017/06/08/build-test-ppc64le-docker-images-intel/)):
+
+     wget http://ftp.unicamp.br/pub/ppc64el/boot2docker/install.sh && chmod +x ./install.sh && ./install.sh -s
+     docker-machine create -d qemu \
+         --qemu-boot2docker-url=/home/ubuntu/.docker/machine/boot2docker.iso \
+         --qemu-memory 8192 \
+         --qemu-cache-mode none \
+         --qemu-arch ppc64le \
+         vm-ppc64le
+
+ Regenerate certs as suggested if it did not work at once.
+
+ Prepare docker machine to run rocksdbjni docker image for ppc64le build:
+
+     eval $(docker-machine env vm-ppc64le)
+     git clone https://github.com/azagrebin/flink-rocksdb-plugins
+     cd flink-rocksdb-plugins
+     git checkout <release tag>
+     docker-machine ssh vm-ppc64le mkdir -p `pwd`
+     docker-machine scp -r . vm-ppc64le:`pwd`
+
+### Build in Windows
+
+Use Windows 64 bit machine (e.g. base AWS Windows instance: 4 cores, 16GB RAM, 40GB storage for build).
+
+Open cmd, install [chocolatey](https://chocolatey.org/install) and run:
+
+    choco install make git.install
+    git clone https://github.com/azagrebin/flink-rocksdb-plugins
+    cd flink-rocksdb-plugins
+    git checkout <release tag>
+    ROCKSDB_VERSION=5.17.2 make win-prepare &:: install useful software and clone rocksdb, might take time
+    make build-win
+
+### Cross-platform build in Mac OSX
+
+Clone rocksdb:
+
+    ROCKSDB_VERSION=5.17.2 make prepare-rocksdb
+
+Install docker and run:
+
+    cp <windows library location>/liblibfrocksdbpluginsjni-linux.dll java/src/main/resources/.
+    cp <ppc64le library location>/liblibfrocksdbpluginsjni-linux-ppc64le.so java/src/main/resources/.
+    make
diff --git a/java/pom.xml b/java/pom.xml
@@ -22,16 +22,28 @@ under the License.
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
-  <groupId>com.data-artisans</groupId>
-  <artifactId>flink-rocksdb-plugin</artifactId>
-  <version>1.0</version>
+  <name>RocksDB plugins for Apache Flink state backend</name>
+  <description>Fat jar with RocksDB plugins for Apache Flink state backend
+    that contains .so files for linux32 and linux64, dylib files for Mac OSX, and a .dll for Windows x64.
+  </description>
+  <url>https://github.com/azagrebin/flink-rocksdb-plugins</url>
 
-  <name>flink-rocksdb-plugin</name>
+  <groupId>com.ververica</groupId>
+  <artifactId>flink-rocksdb-plugins</artifactId>
+  <version>5.17.2-1.0</version>
+
+  <licenses>
+    <license>
+      <name>Apache License 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <maven.compiler.source>1.7</maven.compiler.source>
-    <maven.compiler.target>1.7</maven.compiler.target>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
   </properties>
 
   <dependencies>
@@ -53,84 +65,24 @@ under the License.
       <version>1.7.1</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <version>1.10.19</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
-    <!--<plugins>-->
-      <!--<plugin>-->
-        <!--<groupId>org.codehaus.mojo</groupId>-->
-        <!--<artifactId>native-maven-plugin</artifactId>-->
-        <!--<extensions>true</extensions>-->
-
-        <!--<executions>-->
-          <!--<execution>-->
-            <!--<id>javah</id>-->
-            <!--<phase>generate-sources</phase>-->
-            <!--<configuration>-->
-              <!--<javahProvider>default</javahProvider>-->
-              <!--<javahOutputDirectory>..</javahOutputDirectory>-->
-              <!--<workingDirectory>${basedir}</workingDirectory>-->
-              <!--<javahClassNames>-->
-                <!--<javahClassName>org.rocksdb.FlinkCompactionFilter</javahClassName>-->
-              <!--</javahClassNames>-->
-            <!--</configuration>-->
-            <!--<goals>-->
-              <!--<goal>javah</goal>-->
-            <!--</goals>-->
-          <!--</execution>-->
-        <!--</executions>-->
-
-      <!--</plugin>-->
-    <!--</plugins>-->
-
-    <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
-      <plugins>
-        <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
-        <plugin>
-          <artifactId>maven-clean-plugin</artifactId>
-          <version>3.1.0</version>
-        </plugin>
-        <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
-        <plugin>
-          <artifactId>maven-resources-plugin</artifactId>
-          <version>3.0.2</version>
-        </plugin>
-        <plugin>
-          <artifactId>maven-compiler-plugin</artifactId>
-          <version>3.8.0</version>
-        </plugin>
-        <plugin>
-          <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.22.1</version>
-        </plugin>
-        <plugin>
-          <artifactId>maven-jar-plugin</artifactId>
-          <version>3.0.2</version>
-        </plugin>
-        <plugin>
-          <artifactId>maven-install-plugin</artifactId>
-          <version>2.5.2</version>
-        </plugin>
-        <plugin>
-          <artifactId>maven-deploy-plugin</artifactId>
-          <version>2.8.2</version>
-        </plugin>
-        <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
-        <plugin>
-          <artifactId>maven-site-plugin</artifactId>
-          <version>3.7.1</version>
-        </plugin>
-        <plugin>
-          <artifactId>maven-project-info-reports-plugin</artifactId>
-          <version>3.0.0</version>
-        </plugin>
-      </plugins>
-    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.0</version>
+        <configuration>
+          <compilerArgs>
+            <arg>-h</arg>
+            <arg>../src</arg>
+          </compilerArgs>
+          <source>${maven.compiler.source}</source>
+          <target>${maven.compiler.target}</target>
+          <encoding>${project.build.sourceEncoding}</encoding>
+        </configuration>
+      </plugin>
+    </plugins>
   </build>
 </project>
diff --git a/java/src/main/java/org/rocksdb/FlinkCompactionFilter.java b/java/src/main/java/org/rocksdb/FlinkCompactionFilter.java
@@ -5,11 +5,7 @@
 
 package org.rocksdb;
 
-import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
 
 /**
  * Just a Java wrapper around FlinkCompactionFilter implemented in C++.
@@ -19,11 +15,7 @@
 public class FlinkCompactionFilter
     extends AbstractCompactionFilter<Slice> {
   static {
-    try {
-      FlinkNativeLibraryLoader.getInstance().loadLibraryFromJar(null);
-    } catch (IOException e) {
-      throw new RuntimeException("Unable to load the Flink shared library", e);
-    }
+    FlinkNativeLibraryLoader.load();
   }
 
   public enum StateType {

diff --git a/java/src/main/java/org/rocksdb/FlinkNativeLibraryLoader.java b/java/src/main/java/org/rocksdb/FlinkNativeLibraryLoader.java
@@ -30,10 +30,18 @@ class FlinkNativeLibraryLoader {
     private static final FlinkNativeLibraryLoader instance = new FlinkNativeLibraryLoader();
     private static boolean initialized = false;
 
-    private static final String tempFilePrefix = "librocksdb_plugins";
+    private static final String tempFilePrefix = "lib" + Environment.getJniLibraryName("frocksdbplugins");
     private static final String tempFileSuffix = getJniLibraryExtension();
     private static final String jniLibraryFileName = tempFilePrefix + tempFileSuffix;
 
+    static void load() {
+        try {
+            FlinkNativeLibraryLoader.getInstance().loadLibraryFromJar(null);
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to load the Flink shared library", e);
+        }
+    }
+
     static FlinkNativeLibraryLoader getInstance() {
         return instance;
     }