Introduce XPRESS compresssion on Windows. (facebook#1081)

Comparable with Snappy on comp ratio.
  Implemented using Windows API, does not require external package.
  Avaiable since Windows 8 and server 2012.
  Use -DXPRESS=1 with CMake to enable.

CMakeLists.txt

@@ -82,15 +82,15 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
 set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
 
-add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64)
+add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
 include_directories(${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
 
 set(ROCKSDB_LIBS rocksdblib${ARTIFACT_SUFFIX})
 set(THIRDPARTY_LIBS ${THIRDPARTY_LIBS} gtest)
-set(SYSTEM_LIBS Shlwapi.lib Rpcrt4.lib)
+set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
 
 set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
@@ -156,6 +156,7 @@ set(SOURCES
         port/win/env_win.cc
         port/win/port_win.cc
         port/win/win_logger.cc
+        port/win/xpress_win.cc
         table/adaptive_table_factory.cc
         table/block.cc
         table/block_based_filter_block.cc

db/compaction_job_stats_test.cc

@@ -624,7 +624,10 @@ CompressionType GetAnyCompression() {
     return kBZip2Compression;
   } else if (LZ4_Supported()) {
     return kLZ4Compression;
+  } else if (XPRESS_Supported()) {
+    return kXpressCompression;
   }
+
   return kNoCompression;
 }
 

db/db_test.cc

@@ -3184,8 +3184,8 @@ class ModelDB : public DB {
 
   virtual Status GetUpdatesSince(
       rocksdb::SequenceNumber, unique_ptr<rocksdb::TransactionLogIterator>*,
-      const TransactionLogIterator::ReadOptions& read_options =
-          TransactionLogIterator::ReadOptions()) override {
+      const TransactionLogIterator::ReadOptions&
+          read_options = TransactionLogIterator::ReadOptions()) override {
     return Status::NotSupported("Not supported in Model DB");
   }
 
@@ -3271,7 +3271,8 @@ static bool CompareIterators(int step, DB* model, DB* db,
        ok && miter->Valid() && dbiter->Valid(); miter->Next(), dbiter->Next()) {
     count++;
     if (miter->key().compare(dbiter->key()) != 0) {
-      fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", step,
+      fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
+              step,
               EscapeString(miter->key()).c_str(),
               EscapeString(dbiter->key()).c_str());
       ok = false;
@@ -3474,6 +3475,7 @@ TEST_F(DBTest, BlockBasedTablePrefixIndexTest) {
   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
   options.prefix_extractor.reset(NewFixedPrefixTransform(1));
 
+
   Reopen(options);
   ASSERT_OK(Put("k1", "v1"));
   Flush();
@@ -3672,15 +3674,15 @@ TEST_F(DBTest, TableOptionsSanitizeTest) {
   ASSERT_OK(TryReopen(options));
 }
 
+
+// On Windows you can have either memory mapped file or a file
+// with unbuffered access. So this asserts and does not make
+// sense to run
+#ifndef OS_WIN
 TEST_F(DBTest, MmapAndBufferOptions) {
   Options options = CurrentOptions();
 
-  // If allow_mmap_reads is on allow_os_buffer must also be on
-  // On Windows you can have either memory mapped file or a file
-  // with unbuffered access.
-#ifndef OS_WIN
   options.allow_os_buffer = false;
-#endif
   options.allow_mmap_reads = true;
   ASSERT_NOK(TryReopen(options));
 
@@ -3695,6 +3697,7 @@ TEST_F(DBTest, MmapAndBufferOptions) {
   options.allow_os_buffer = true;
   ASSERT_OK(TryReopen(options));
 }
+#endif
 
 TEST_F(DBTest, ConcurrentMemtableNotSupported) {
   Options options = CurrentOptions();
@@ -4948,10 +4951,12 @@ TEST_F(DBTest, EncodeDecompressedBlockSizeTest) {
   // iter 1 -- bzip2
   // iter 2 -- lz4
   // iter 3 -- lz4HC
+  // iter 4 -- xpress
   CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
-                                    kLZ4Compression,  kLZ4HCCompression};
-  for (int iter = 0; iter < 4; ++iter) {
-    if (!CompressionTypeSupported(compressions[iter])) {
+                                    kLZ4Compression,  kLZ4HCCompression,
+                                    kXpressCompression};
+  for (auto comp : compressions) {
+    if (!CompressionTypeSupported(comp)) {
       continue;
     }
     // first_table_version 1 -- generate with table_version == 1, read with
@@ -4966,7 +4971,7 @@ TEST_F(DBTest, EncodeDecompressedBlockSizeTest) {
       Options options = CurrentOptions();
       options.table_factory.reset(NewBlockBasedTableFactory(table_options));
       options.create_if_missing = true;
-      options.compression = compressions[iter];
+      options.compression = comp;
       DestroyAndReopen(options);
 
       int kNumKeysWritten = 100000;
@@ -5809,18 +5814,19 @@ TEST_F(DBTest, LastWriteBufferDelay) {
 
 TEST_F(DBTest, FailWhenCompressionNotSupportedTest) {
   CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
-                                    kLZ4Compression, kLZ4HCCompression};
-  for (int iter = 0; iter < 4; ++iter) {
-    if (!CompressionTypeSupported(compressions[iter])) {
+                                    kLZ4Compression,  kLZ4HCCompression,
+                                    kXpressCompression};
+  for (auto comp : compressions) {
+    if (!CompressionTypeSupported(comp)) {
       // not supported, we should fail the Open()
       Options options = CurrentOptions();
-      options.compression = compressions[iter];
+      options.compression = comp;
       ASSERT_TRUE(!TryReopen(options).ok());
       // Try if CreateColumnFamily also fails
       options.compression = kNoCompression;
       ASSERT_OK(TryReopen(options));
       ColumnFamilyOptions cf_options(options);
-      cf_options.compression = compressions[iter];
+      cf_options.compression = comp;
       ColumnFamilyHandle* handle;
       ASSERT_TRUE(!db_->CreateColumnFamily(cf_options, "name", &handle).ok());
     }

db/db_wal_test.cc

@@ -584,8 +584,11 @@ class RecoveryTestHelper {
 
     int fd = open(filename.c_str(), O_RDWR);
 
+    // On windows long is 32-bit
+    ASSERT_LE(offset, std::numeric_limits<long>::max());
+
     ASSERT_GT(fd, 0);
-    ASSERT_EQ(offset, lseek(fd, offset, SEEK_SET));
+    ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET));
 
     void* buf = alloca(len);
     memset(buf, 'a', len);

include/rocksdb/options.h

@@ -61,6 +61,7 @@ enum CompressionType : char {
   kBZip2Compression = 0x3,
   kLZ4Compression = 0x4,
   kLZ4HCCompression = 0x5,
+  kXpressCompression = 0x6,
   // zstd format is not finalized yet so it's subject to changes.
   kZSTDNotFinalCompression = 0x40,
 };

port/win/port_win.h

@@ -56,23 +56,6 @@ typedef SSIZE_T ssize_t;
 
 #define __attribute__(A)
 
-#ifdef ZLIB
-#include <zlib.h>
-#endif
-
-#ifdef BZIP2
-#include <bzlib.h>
-#endif
-
-#if defined(LZ4)
-#include <lz4.h>
-#include <lz4hc.h>
-#endif
-
-#ifdef SNAPPY
-#include <snappy.h>
-#endif
-
 // Thread local storage on Linux
 // There is thread_local in C++11
 #ifndef __thread