Add -data_max_factor_levels and -chunk_bits to hadoop command line.

panzelon · Jan 12, 2015 · e0cb1a8 · e0cb1a8
1 parent fa521e4
commit e0cb1a8
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 0 deletions.
diff --git a/hadoop/src/main/java/water/hadoop/h2odriver.java b/hadoop/src/main/java/water/hadoop/h2odriver.java
@@ -47,6 +47,8 @@ public class h2odriver extends Configured implements Tool {
   static int cloudFormationTimeoutSeconds = DEFAULT_CLOUD_FORMATION_TIMEOUT_SECONDS;
   static int nthreads = -1;
   static int basePort = -1;
+  static int chunk_bits;
+  static int data_max_factor_levels;
   static boolean beta = false;
   static boolean enableRandomUdpDrop = false;
   static boolean enableExceptions = false;
@@ -389,6 +391,8 @@ static void usage() {
                     "          -n | -nodes <number of H2O nodes (i.e. mappers) to create>\n" +
                     "          [-nthreads <maximum typical worker threads, i.e. cpus to use>]\n" +
                     "          [-baseport <starting HTTP port for H2O nodes; default is 54321>]\n" +
+                    "          [-chunk_bits <bits per chunk (e.g., 22 for 4MB chunks)>]\n" +
+                    "          [-data_max_factor_levels <max. number of factors per column (e.g., 100,000)>]\n" +
                     "          [-ea]\n" +
                     "          [-verbose:gc]\n" +
                     "          [-XX:+PrintGCDetails]\n" +
@@ -543,6 +547,14 @@ else if (s.equals("-nthreads")) {
         i++; if (i >= args.length) { usage(); }
         nthreads = Integer.parseInt(args[i]);
       }
+      else if (s.equals("-chunk_bits")) {
+        i++; if (i >= args.length) { usage(); }
+        chunk_bits = Integer.parseInt(args[i]);
+      }
+      else if (s.equals("-data_max_factor_levels")) {
+        i++; if (i >= args.length) { usage(); }
+        data_max_factor_levels = Integer.parseInt(args[i]);
+      }
       else if (s.equals("-baseport")) {
         i++; if (i >= args.length) { usage(); }
         basePort = Integer.parseInt(args[i]);
@@ -912,6 +924,12 @@ private int run2(String[] args) throws Exception {
     if (beta) {
         conf.set(h2omapper.H2O_BETA_KEY, "-beta");
     }
+    if (chunk_bits > 0) {
+      conf.set(h2omapper.H2O_CHUNKBITS_KEY, Integer.toString(chunk_bits));
+    }
+    if (data_max_factor_levels > 0) {
+      conf.set(h2omapper.H2O_DATAMAXFACTORLEVELS_KEY, Integer.toString(data_max_factor_levels));
+    }
     if (enableRandomUdpDrop) {
       conf.set(h2omapper.H2O_RANDOM_UDP_DROP_KEY, "-random_udp_drop");
     }

diff --git a/hadoop/src/main/java/water/hadoop/h2omapper.java b/hadoop/src/main/java/water/hadoop/h2omapper.java
@@ -27,6 +27,8 @@ public class h2omapper extends Mapper<Text, Text, Text, Text> {
   final static public String H2O_BETA_KEY = "h2o.beta";
   final static public String H2O_RANDOM_UDP_DROP_KEY = "h2o.random.udp.drop";
   final static public String H2O_NTHREADS_KEY = "h2o.nthreads";
+  final static public String H2O_CHUNKBITS_KEY = "h2o.chunk.bits";
+  final static public String H2O_DATAMAXFACTORLEVELS_KEY = "h2o.data.max.factor.levels";
   final static public String H2O_BASE_PORT_KEY = "h2o.baseport";
   final static public String H2O_LICENSE_DATA_KEY = "h2o.license.data";
   final static public String H2O_HADOOP_VERSION = "h2o.hadoop.version";
@@ -360,6 +362,8 @@ private int run2(Context context) throws IOException, InterruptedException {
     String driverIp = conf.get(H2O_DRIVER_IP_KEY);
     String driverPortString = conf.get(H2O_DRIVER_PORT_KEY);
     String network = conf.get(H2O_NETWORK_KEY);
+    String chunkBitsString = conf.get(H2O_CHUNKBITS_KEY);
+    String dataMaxFactorLevelsString = conf.get(H2O_DATAMAXFACTORLEVELS_KEY);
     String nthreadsString = conf.get(H2O_NTHREADS_KEY);
     String basePortString = conf.get(H2O_BASE_PORT_KEY);
     String betaString = conf.get(H2O_BETA_KEY);
@@ -401,6 +405,20 @@ private int run2(Context context) throws IOException, InterruptedException {
         argsList.add(Integer.toString(basePort));
       }
     }
+    if (dataMaxFactorLevelsString != null) {
+      if (dataMaxFactorLevelsString.length() > 0) {
+        argsList.add("-max_data_factor_levels");
+        int dataMaxFactorLevels = Integer.parseInt(dataMaxFactorLevelsString);
+        argsList.add(Integer.toString(dataMaxFactorLevels));
+      }
+    }
+    if (chunkBitsString != null) {
+      if (chunkBitsString.length() > 0) {
+        argsList.add("-chunk_bits");
+        int chunkBits = Integer.parseInt(chunkBitsString);
+        argsList.add(Integer.toString(chunkBits));
+      }
+    }
     if (betaString != null) {
       if (betaString.length() > 0) {
         argsList.add(betaString);