new:dev: Upgrade thrift & maven-thrift-plugin. Make tests relocatable (…

…qubole#46) Add gitignore file with common patterns for Java and Intellij Upgraded thrift from 0.9.0 to 0.9.3. The main motivation is that 0.9.3 can be installed through brew on mac. I tried to upgrade to thrift 0.10.0 but there were test failures. Another improvement is that tests assumed specific directory structures internal to Qubole. The test directories are now configured relative to `java.io.tmpdir`.
heypinch · Jul 17, 2017 · d7c0070 · d7c0070
1 parent c7d1edf
commit d7c0070
Show file tree

Hide file tree

Showing 8 changed files with 131 additions and 16 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,23 @@
+# Compiled class file
+*.class
+#
+# # Log file
+*.log
+#
+# # Package Files #
+*.jar
+*.war
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+# hs_err_pid*
+#
+# IntelliJ files
+.idea/*
+*.iml
+
+# Maven
+target
diff --git a/README.md b/README.md
@@ -81,9 +81,21 @@ presto:default> select * from jmx.jmx."rubix:name=stats";
 
 ### Building
 
-mvn clean install
+#### Setup Thrift
+Note that you will need thrift-0.9.3 installed
+
+    # Mac OS X/Brew instructions
+
+    ## Installs thrift 0.10.0
+    brew install thrift
+
+    ### Install thrift 0.9.3
+    brew unlink thrift
+    brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/9d524e4850651cfedd64bc0740f1379b533f607d/Formula/thrift.rb
+
+#### Compile and install
+    mvn clean install
 
-> Note that you will need thrift-0.9.0 installed
 
 ### Need Help?
 You can post your queries to https://groups.google.com/forum/#!forum/rubix-users
diff --git a/rubix-spi/pom.xml b/rubix-spi/pom.xml
@@ -20,7 +20,7 @@
         <dependency>
             <groupId>org.apache.thrift</groupId>
             <artifactId>libthrift</artifactId>
-            <version>0.9.2</version>
+            <version>0.9.3</version>
         </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
@@ -74,9 +74,10 @@
             <plugin>
                 <groupId>org.apache.thrift.tools</groupId>
                 <artifactId>maven-thrift-plugin</artifactId>
-                <version>0.1.10</version>
+                <version>0.1.11</version>
                 <configuration>
                     <thriftExecutable>/usr/local/bin/thrift</thriftExecutable>
+                    <generator>java</generator>
                 </configuration>
                 <executions>
                     <execution>
@@ -102,7 +103,7 @@
                         <configuration>
                             <artifactSet>
                                 <includes>
-                                    -<include>org.apache.thrift:libthrift</include>
+                                    <include>org.apache.thrift:libthrift</include>
                                     <include>com.google.guava:guava</include>
                                 </includes>
                             </artifactSet>
@@ -123,10 +124,4 @@
             </plugin>
         </plugins>
     </build>
-    <pluginRepositories>
-        <pluginRepository>
-            <id>dtrott</id>
-            <url>http://maven.davidtrott.com/repository</url>
-        </pluginRepository>
-    </pluginRepositories>
 </project>
diff --git a/rubix-spi/src/main/java/com/qubole/rubix/spi/CacheConfig.java b/rubix-spi/src/main/java/com/qubole/rubix/spi/CacheConfig.java
@@ -18,6 +18,8 @@
 import com.google.common.hash.HashCode;
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 
@@ -84,6 +86,8 @@ public class CacheConfig
     public static int socketReadTimeOutDefault = 30000; // In milliseconds.
     private static int diskMonitorInterval = 10; // in seconds
 
+    private static final Log log = LogFactory.getLog(CacheConfig.class.getName());
+
     private CacheConfig()
     {
     }
@@ -160,6 +164,7 @@ public HashMap<Integer, String> get()
                 List<String> dirPrefixList = getDirPrefixList(conf);
                 for (String dirPrefix : dirPrefixList) {
                     for (int i = 0; i < maxDisksConf; ++i) {
+                        log.debug("Checking " + dirPrefix + i);
                         if (exists(dirPrefix + i)) {
                             File dir = new File(dirPrefix + i + fileCacheDirSuffixConf);
                             dir.mkdir();

diff --git a/rubix-tests/pom.xml b/rubix-tests/pom.xml
@@ -77,6 +77,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
+                    <workingDirectory>${project.build.testOutputDirectory}</workingDirectory>
                     <systemPropertyVariables>
                         <log4j.configuration>file:${project.build.testOutputDirectory}/log4j-surefire.properties</log4j.configuration>
                     </systemPropertyVariables>

diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/DeleteFileVisitor.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/DeleteFileVisitor.java
@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2016. Qubole Inc
+ * Licensed under the Apache License, Version 2.0 (the License);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package com.qubole.rubix.tests;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+
+/**
+ * Created by rvenkatesh on 7/17/17.
+ */
+class DeleteFileVisitor extends SimpleFileVisitor<Path> {
+  @Override
+  public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+    Files.delete(file);
+    return FileVisitResult.CONTINUE;
+  }
+
+  @Override
+  public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+    Files.delete(dir);
+    return FileVisitResult.CONTINUE;
+  }
+}
diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestCachingInputStream.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestCachingInputStream.java
@@ -30,13 +30,17 @@
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
 
 
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Arrays;
 
 import static org.testng.AssertJUnit.assertTrue;
@@ -47,13 +51,28 @@
 public class TestCachingInputStream
 {
     int blockSize = 100;
-    String backendFileName = "/tmp/backendFile";
+    private final static String testDirectoryPrefix = System.getProperty("java.io.tmpdir") + "TestCachingInputStream/";
+    String backendFileName = testDirectoryPrefix + "backendFile";
     Path backendPath = new Path("file://" + backendFileName.substring(1));
 
     CachingInputStream inputStream;
 
+    private final static String testDirectory = testDirectoryPrefix + "dir0";
+
     private static final Log log = LogFactory.getLog(TestCachingInputStream.class.getName());
 
+    @BeforeClass
+    public static void setupClass() throws IOException {
+        log.info(testDirectory);
+        Files.createDirectories(Paths.get(testDirectory));
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws IOException {
+        log.info("Deleting files in " + testDirectory);
+        Files.walkFileTree(Paths.get(testDirectory), new DeleteFileVisitor());
+        Files.deleteIfExists(Paths.get(testDirectory));
+    }
 
     @BeforeMethod
     public void setup()
@@ -65,7 +84,7 @@ public void setup()
         conf.setBoolean(CacheConfig.DATA_CACHE_STRICT_MODE, true);
         conf.setInt(CacheConfig.dataCacheBookkeeperPortConf, 3456);
         conf.setInt(CacheConfig.localServerPortConf, 2222);
-        conf.set("hadoop.cache.data.dirprefix.list", "/tmp/ephemeral");
+        conf.set(CacheConfig.dataCacheDirprefixesConf, testDirectoryPrefix + "dir");
         Thread server = new Thread()
         {
             public void run()
@@ -116,6 +135,7 @@ public void cleanup()
         BookKeeperServer.stopServer();
         LocalDataTransferServer.stopServer();
         Configuration conf = new Configuration();
+        conf.set(CacheConfig.dataCacheDirprefixesConf, testDirectoryPrefix + "dir");
         inputStream.close();
         File file = new File(backendFileName);
         file.delete();
@@ -186,7 +206,7 @@ public void testChunkCachingAndEviction()
         //6. Close existing stream and start a new one to get the new lastModifiedDate of backend file
         inputStream.close();
         Configuration conf = new Configuration();
-        conf.set("hadoop.cache.data.dirprefix.list", "/tmp/ephemeral");
+        conf.set(CacheConfig.dataCacheDirprefixesConf, testDirectoryPrefix + "dir");
         createCachingStream(conf);
         log.info("New stream started");
 

diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java
@@ -35,25 +35,46 @@
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 
 import static org.testng.AssertJUnit.assertTrue;
 
 public class TestNonLocalReadRequestChain
 {
     int blockSize = 100;
-    String backendFileName = "/tmp/backendFile";
+    private final static String testDirectoryPrefix = System.getProperty("java.io.tmpdir") + "TestNonLocalReadRequestChain/";
+    String backendFileName = testDirectoryPrefix + "backendFile";
     Path backendPath = new Path("testfile:/" + backendFileName);
     File backendFile = new File(backendFileName);
     final Configuration conf = new Configuration();
     Thread localDataTransferServer;
 
+    private final static String testDirectory = testDirectoryPrefix + "dir0";
+
     NonLocalReadRequestChain nonLocalReadRequestChain;
     private static final Log log = LogFactory.getLog(TestNonLocalReadRequestChain.class);
 
+    @BeforeClass
+    public static void setupClass() throws IOException {
+      log.info(testDirectory);
+      Files.createDirectories(Paths.get(testDirectory));
+    }
+
+
+    @AfterClass
+    public static void tearDownClass() throws IOException {
+      log.info("Deleting files in " + testDirectory);
+      Files.walkFileTree(Paths.get(testDirectory), new DeleteFileVisitor());
+      Files.deleteIfExists(Paths.get(testDirectory));
+    }
+
     @BeforeMethod
     public void setup()
             throws Exception
@@ -62,7 +83,7 @@ public void setup()
         conf.setInt(CacheConfig.dataCacheBookkeeperPortConf, 3456);
         conf.setInt(CacheConfig.localServerPortConf, 2222);
         conf.setInt(CacheConfig.blockSizeConf, blockSize);
-        conf.set("hadoop.cache.data.dirprefix.list", "/tmp/ephemeral");
+        conf.set(CacheConfig.dataCacheDirprefixesConf, testDirectoryPrefix + "dir");
         localDataTransferServer = new Thread()
         {
             public void run()