Moving toward fixing JSON serialization of longs (e.g., seed).

.classpath

@@ -10,7 +10,6 @@
 	<classpathentry kind="lib" path="lib/apache/httpcore-4.1.jar" sourcepath="lib/apache/httpcore-4.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/junit/junit-4.11.jar" sourcepath="lib/junit/junit-4.11-sources.jar"/>
 	<classpathentry kind="lib" path="lib/apache/guava-12.0.1.jar" sourcepath="lib/apache/guava-12.0.1-sources.jar"/>
-	<classpathentry kind="lib" path="lib/gson/gson-2.2.2.jar" sourcepath="lib/gson/gson-2.2.2-sources.jar"/>
 	<classpathentry kind="lib" path="lib/poi/poi-3.8-20120326.jar" sourcepath="lib/poi/poi-3.8-sources.jar"/>
 	<classpathentry kind="lib" path="lib/poi/poi-ooxml-3.8-20120326.jar"/>
 	<classpathentry kind="lib" path="lib/poi/poi-ooxml-schemas-3.8-20120326.jar"/>
@@ -35,7 +34,8 @@
 	<classpathentry kind="lib" path="lib/jogamp/gluegen-rt-natives-linux-amd64.jar"/>
 	<classpathentry kind="lib" path="lib/jogamp/gluegen-rt.jar" sourcepath="lib/jogamp/gluegen-rt-sources.jar"/>
 	<classpathentry kind="lib" path="lib/joda/joda-time-2.3.jar"/>
-  <classpathentry kind="lib" path="lib/json/JSON-java.jar"/>
-  <classpathentry kind="lib" path="lib/tachyon/tachyon-0.4.1-thrift.jar"/>
+	<classpathentry kind="lib" path="lib/json/JSON-java.jar"/>
+	<classpathentry kind="lib" path="lib/tachyon/tachyon-0.4.1-thrift.jar"/>
+	<classpathentry kind="lib" path="lib/gson/gson-2.3.jar"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

h2o-samples/.classpath

@@ -4,8 +4,9 @@
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/h2o"/>
 	<classpathentry kind="lib" path="/h2o/lib/jets3t/commons-httpclient-3.1.jar" sourcepath="/h2o/lib/jets3t/commons-httpclient-3.1-sources.jar"/>
-	<classpathentry kind="lib" path="/h2o/lib/gson/gson-2.2.2.jar" sourcepath="/h2o/lib/gson/gson-2.2.2-sources.jar"/>
+	<classpathentry kind="lib" path="/lib/gson/gson-2.2.2.jar" sourcepath="/lib/gson/gson-2.2.2-sources.jar"/>
 	<classpathentry kind="lib" path="/h2o/lib/joda/joda-time-2.3.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+	<classpathentry kind="lib" path="/h2o/lib/gson/gson-2.3.jar"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

h2o-samples/src/main/java/samples/expert/WebAPI.java

@@ -1,17 +1,23 @@
 package samples.expert;
 
-import java.io.*;
-
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.internal.Streams;
+import com.google.gson.stream.JsonWriter;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.multipart.*;
-
+import org.apache.commons.httpclient.methods.multipart.FilePart;
+import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
+import org.apache.commons.httpclient.methods.multipart.Part;
 import water.util.Utils;
 
-import com.google.gson.*;
-import com.google.gson.internal.Streams;
-import com.google.gson.stream.JsonWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+
 
 /**
  * Invokes H2O functionality through the Web API.

pom.xml

@@ -105,7 +105,7 @@
     <dependency>
       <groupId>com.google.code.gson</groupId>
       <artifactId>gson</artifactId>
-      <version>2.2.2</version>
+      <version>2.3</version>
       <scope>compile</scope>
     </dependency>
     <!-- ==Jama is given explicitely - it was modified and compiled by @tomas == -->

prj.el

@@ -2,15 +2,15 @@
 (jde-set-variables
  '(jde-javadoc-gen-destination-directory "./doc")
  '(jde-run-working-directory "$DESK/h2o")
- '(jde-run-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.2.2.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/jets3t/commons-httpclient-3.1.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar" "./lib/tachyon/tachyon-0.4.1-thrift.jar")))
+ '(jde-run-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.3.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/jets3t/commons-httpclient-3.1.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar" "./lib/tachyon/tachyon-0.4.1-thrift.jar")))
  '(jde-run-executable-args nil)
  '(jde-run-option-debug nil)
  '(jde-run-option-vm-args (quote ("-XX:+PrintGC")))
  '(jde-compile-option-directory "./target/classes")
  '(jde-run-option-application-args (quote ("-beta" "-mainClass" "org.junit.runner.JUnitCore" "water.parser.DatasetCornerCasesTest")))
  '(jde-debugger (quote ("JDEbug")))
  '(jde-compile-option-source (quote ("1.6")))
- '(jde-compile-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/commons-math3-3.2.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.2.2.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar")))
+ '(jde-compile-option-classpath (quote ("./target/classes" "./lib/javassist.jar" "./lib/hadoop/cdh4/hadoop-common.jar" "./lib/hadoop/cdh4/hadoop-auth.jar" "./lib/hadoop/cdh4/slf4j-api-1.6.1.jar" "./lib/hadoop/cdh4/slf4j-nop-1.6.1.jar" "./lib/hadoop/cdh4/hadoop-hdfs.jar" "./lib/hadoop/cdh4/protobuf-java-2.4.0a.jar" "./lib/apache/commons-codec-1.4.jar" "./lib/apache/commons-configuration-1.6.jar" "./lib/apache/commons-lang-2.4.jar" "./lib/apache/commons-logging-1.1.1.jar" "./lib/apache/commons-math3-3.2.jar" "./lib/apache/httpclient-4.1.1.jar" "./lib/apache/httpcore-4.1.jar" "./lib/junit/junit-4.11.jar" "./lib/apache/guava-12.0.1.jar" "./lib/gson/gson-2.3.jar" "./lib/poi/poi-3.8-20120326.jar" "./lib/poi/poi-ooxml-3.8-20120326.jar" "./lib/poi/poi-ooxml-schemas-3.8-20120326.jar" "./lib/poi/dom4j-1.6.1.jar" "./lib/Jama/Jama.jar" "./lib/s3/aws-java-sdk-1.3.27.jar" "./lib/log4j/log4j-1.2.15.jar" "./lib/joda/joda-time-2.3.jar")))
  '(jde-db-option-classpath (quote ("$DESK/Dropbox/Sris and Cliff/H2O/classes")))
  '(jde-run-option-enable-assertions "Everywhere")
  '(jde-compile-option-sourcepath (quote ("./src")))

py/testdir_multi_jvm/test_model_management.py

@@ -303,7 +303,8 @@ def create_models(self, frame_keys):
             'ignored_cols_by_name': 'IsDepDelayed_REC, IsDepDelayed_REC_recoded', 
             'ntrees': 5,
             'max_depth': 2,
-            'classification': 1
+            'classification': 1,
+            'seed': 1234567890123456789L
         }
         rf_AirlinesTrain_1 = node.random_forest(airlines_train_hex, **rf_AirlinesTrain_1_params)
         durations['rf_AirlinesTrain_binary_1'] = time.time() * 1000 - before
@@ -320,7 +321,8 @@ def create_models(self, frame_keys):
             'ignored_cols_by_name': 'IsDepDelayed_REC, IsDepDelayed_REC_recoded', 
             'ntrees': 50,
             'max_depth': 10,
-            'classification': 1
+            'classification': 1,
+            'seed': 1234567890123456789L
         }
         rf_AirlinesTrain_2 = node.random_forest(airlines_train_hex, **rf_AirlinesTrain_2_params)
         durations['rf_AirlinesTrain_binary_2'] = time.time() * 1000 - before
@@ -338,6 +340,7 @@ def create_models(self, frame_keys):
             'ntrees': 5,
             'max_depth': 2,
             'classification': 1,
+            'seed': 1234567890123456789L,
             'n_folds': 3
         }
         rf_AirlinesTrain_3fold = node.random_forest(airlines_train_hex, **rf_AirlinesTrain_3fold_params)
@@ -356,7 +359,8 @@ def create_models(self, frame_keys):
             'ntrees': 50,
             'max_depth': 10,
             'classification': 1,
-            'importance': 1
+            'importance': 1,
+            'seed': 1234567890123456789L
         }
         speedrf_AirlinesTrain_1 = node.speedrf(airlines_train_hex, **speedrf_AirlinesTrain_1_params)
         durations['speedrf_AirlinesTrain_binary_1'] = time.time() * 1000 - before
@@ -375,6 +379,7 @@ def create_models(self, frame_keys):
             'max_depth': 10,
             'classification': 1,
             'importance': 1,
+            'seed': 1234567890123456789L,
             'n_folds': 3
         }
         speedrf_AirlinesTrain_3fold = node.speedrf(airlines_train_hex, **speedrf_AirlinesTrain_3fold_params)
@@ -481,7 +486,8 @@ def create_models(self, frame_keys):
             'ignored_cols_by_name': None, 
             'ntrees': 10,
             'max_depth': 5,
-            'classification': 1
+            'classification': 1,
+            'seed': 1234567890123456789L
         }
         rf_Prostate_1 = node.random_forest(prostate_hex, **rf_Prostate_1_params)
         durations['rf_Prostate_binary_1'] = time.time() * 1000 - before
@@ -498,7 +504,8 @@ def create_models(self, frame_keys):
             'ntrees': 50,
             'max_depth': 10,
             'classification': 1,
-            'importance': 1
+            'importance': 1,
+            'seed': 1234567890123456789L
         }
         speedrf_Prostate_1 = node.speedrf(prostate_hex, **speedrf_Prostate_1_params)
         num_models = num_models + 1
@@ -523,8 +530,11 @@ def create_models(self, frame_keys):
         num_models = num_models + 1
         h2o_glm.simpleCheckGLM(self, glm_Prostate_regression_1, None, **glm_Prostate_regression_1_params)
 
+
+        # Done building models!
         # We were getting different results for each node.  Bad, bad bad. . .
-        print "Checking " + str(len(h2o.nodes)) + " nodes for models: "
+        print "########################################################"
+        print "Checking " + str(len(h2o.nodes)) + " nodes for " + str(num_models) + " models: "
         for a_node in h2o.nodes:
             print "  " + a_node.http_addr + ":" + str(a_node.port)
 
@@ -549,6 +559,13 @@ def create_models(self, frame_keys):
                 else:
                     expected = durations[key[0:idx]]
                 self.assertTrue(value['training_duration_in_ms'] < expected, "Expected training duration as computed by the server (" + str(value['training_duration_in_ms']) + ") to be less than we compute in the test  (" + str(expected) + ") for model: " + key)
+
+                self.assertKeysExistAndNonNull(value, "", ['expert_parameters'])
+                # TODO: put back when Long serialization is fixed (probably not until h2o-dev)
+                # if 'seed' in value['expert_parameters']:
+                #     self.assertEquals(long(value['expert_parameters']['seed']), 1234567890123456789L, "Seed incorrect for model: " + key + ".  Expected: 1234567890123456789; got: " + str(long(value['expert_parameters']['seed'])))
+                # if '_seed' in value['expert_parameters']:
+                #     self.assertEquals(long(value['expert_parameters']['_seed']), 1234567890123456789L, "Seed incorrect for model: " + key + ".  Expected: 1234567890123456789; got: " + str(long(value['expert_parameters']['_seed'])))
         self.assertNotEqual(found_problem, True, "Missing models on at least one node.")
 
 

src/main/java/water/Request2.java

@@ -1,14 +1,13 @@
 package water;
 
+import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import hex.GridSearch;
 import water.api.DocGen;
 import water.api.Request;
 import water.api.RequestArguments;
-import water.api.RequestBuilders.Response;
 import water.api.RequestServer.API_VERSION;
-import water.fvec.Frame;
 import water.util.Log;
 import water.util.Utils;
 
@@ -552,6 +551,17 @@ public JsonObject toJSON() {
     return jo;
   }
 
+  public JsonObject toJSON(Set<String> whitelist) {
+    JsonObject jo = toJSON();
+
+    for (Map.Entry<String , JsonElement> entry : jo.entrySet()) {
+      String key = entry.getKey();
+      if (! whitelist.contains(key))
+        jo.remove(key);
+    }
+    return jo;
+  }
+
   @Override
   public String toString() {
     return GSON_BUILDER.toJson(toJSON());

src/main/java/water/util/Utils.java

@@ -1,8 +1,16 @@
 package water.util;
 
-import hex.rng.*;
+import hex.rng.H2ORandomRNG;
 import hex.rng.H2ORandomRNG.RNGKind;
 import hex.rng.H2ORandomRNG.RNGType;
+import hex.rng.MersenneTwisterRNG;
+import hex.rng.XorShiftRNG;
+import sun.misc.Unsafe;
+import water.*;
+import water.api.DocGen;
+import water.api.DocGen.FieldDoc;
+import water.fvec.ParseDataset2.Compression;
+import water.nbhm.UtilUnsafe;
 
 import java.io.*;
 import java.net.Socket;
@@ -11,15 +19,12 @@
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.zip.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
 
 import static java.lang.Double.isNaN;
-import sun.misc.Unsafe;
-import water.*;
-import water.api.DocGen;
-import water.api.DocGen.FieldDoc;
-import water.nbhm.UtilUnsafe;
-import water.fvec.ParseDataset2.Compression;
 
 public class Utils {
   /** Returns the index of the largest value in the array.
@@ -813,6 +818,16 @@ public static class IcedDouble extends Iced {
     @Override public int hashCode() { return (int)Double.doubleToLongBits(_val); }
     @Override public String toString() { return Double.toString(_val); }
   }
+  public static class IcedString extends Iced {
+    public final String _val;
+    public IcedString(String v){_val = v;}
+    @Override public boolean equals( Object o ) {
+      if( !(o instanceof IcedString) ) return false;
+      return ((IcedString)o)._val.equals(_val);
+    }
+    @Override public int hashCode() { return _val.hashCode(); }
+    @Override public String toString() { return _val; }
+  }
   public static class IcedBitSet extends Iced {
     public final byte[] _val;
     public final int _nbits;