start of a parser-exception test

nuoncul · Jul 16, 2014 · a0b78c6 · a0b78c6
1 parent 50bc74b
commit a0b78c6
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 9 deletions.
diff --git a/prj.el b/prj.el
@@ -2,12 +2,12 @@
 (jde-set-variables
  '(jde-javadoc-gen-destination-directory "./doc")
  '(jde-run-working-directory "$DESK/h2o")
- '(jde-run-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.2.2.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar")))
+ '(jde-run-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.2.2.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar" "./lib/tachyon/tachyon-0.4.1-thrift.jar")))
  '(jde-run-executable-args nil)
  '(jde-run-option-debug nil)
  '(jde-run-option-vm-args (quote ("-XX:+PrintGC")))
  '(jde-compile-option-directory "./target/classes")
- '(jde-run-option-application-args (quote ("-beta" "-mainClass" "org.junit.runner.JUnitCore" "water.fvec.ParseTimeTest")))
+ '(jde-run-option-application-args (quote ("-beta" "-mainClass" "org.junit.runner.JUnitCore" "water.parser.ParseFolderTest")))
  '(jde-debugger (quote ("JDEbug")))
  '(jde-compile-option-source (quote ("1.6")))
  '(jde-compile-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/commons-math3-3.2.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.2.2.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar")))

diff --git a/src/main/java/water/fvec/Chunk.java b/src/main/java/water/fvec/Chunk.java
@@ -14,7 +14,7 @@ public abstract class Chunk extends Iced implements Cloneable {
   public int _len;            // Number of elements in this chunk
   protected Chunk _chk2;      // Normally==null, changed if chunk is written to
   public Vec _vec;            // Owning Vec; filled after AutoBuffer.read
-  byte[] _mem; // Short-cut to the embedded memory; WARNING: holds onto a large array
+  public byte[] _mem; // Short-cut to the embedded memory; WARNING: holds onto a large array
 
   public final boolean readable( ) { return _vec.readable(); }
   public final boolean writable( ) { return _vec.writable(); }

diff --git a/src/main/java/water/util/Utils.java b/src/main/java/water/util/Utils.java
@@ -666,11 +666,10 @@ public static ValueArray loadAndParseKey(Key okey, String path) {
   }
 
   public static Compression guessCompressionMethod(byte [] bits){
-    AutoBuffer ab = new AutoBuffer(bits);
     // Look for ZIP magic
-    if( bits.length > ZipFile.LOCHDR && ab.get4(0) == ZipFile.LOCSIG )
+    if( bits.length > ZipFile.LOCHDR && UDP.get4(bits,0) == ZipFile.LOCSIG )
       return Compression.ZIP;
-    if( bits.length > 2 && ab.get2(0) == GZIPInputStream.GZIP_MAGIC )
+    if( bits.length > 2 && UDP.get2(bits,0) == GZIPInputStream.GZIP_MAGIC )
       return Compression.GZIP;
     return Compression.NONE;
   }

diff --git a/src/test/java/water/parser/ParseFolderTest.java b/src/test/java/water/parser/ParseFolderTest.java
@@ -1,14 +1,14 @@
 package water.parser;
 
 import static org.junit.Assert.assertTrue;
-
 import org.junit.Test;
-
+import java.io.File;
 import water.*;
+import water.fvec.*;
 
 public class ParseFolderTest extends TestUtil {
 
-  @Test public void  testProstate(){
+  @Test public void testProstate() {
     Key k1 = null,k2 = null;
     try {
       k2 = loadAndParseFolder("multipart.hex","smalldata/parse_folder_test" );
@@ -21,4 +21,25 @@ public class ParseFolderTest extends TestUtil {
       Lockable.delete(k2);
     }
   }
+
+  @Test public void testParserRecoversFromException() {
+    try {
+      Key fkey0 = NFSFileVec.make(new File("smalldata/parse_folder_test/prostate_0.csv"));
+      Key fkey1 = NFSFileVec.make(new File("smalldata/parse_folder_test/prostate_1.csv"));
+//      Key fkey2 = NFSFileVec.make(new File("smalldata/parse_folder_test/prostate_2.csv"));
+      // Now "break" one of the files.
+      Vec vec = DKV.get(fkey1).get();
+      Chunk chk = vec.chunkForChunkIdx(0); // Load the chunk (which otherwise loads only lazily)
+      chk._len = 9999;
+      chk._mem = null;
+
+      Key okey = Key.make("junk.hex");
+      Frame fr = ParseDataset2.parse(okey, new Key[]{fkey0,fkey1});
+      System.out.println("Number of chunks: "+fr.anyVec().nChunks());
+      fr.delete();
+
+    } finally {
+    }
+  }
+
 }