Switched JUnit to Java only & added EC2 util

build.sh

@@ -34,7 +34,9 @@ DEPENDENCIES="${JAR_ROOT}/jama/*${SEP}${JAR_ROOT}/apache/*${SEP}${JAR_ROOT}/juni
 
 DEFAULT_HADOOP_VERSION="1.0.0"
 OUTDIR="target"
+JAR_FILE="${OUTDIR}/h2o.jar"
 
+JAVA=`which java`||echo 'Missing java, please install jdk'
 JAVAC=`which javac`||echo 'Missing javac, please install jdk'
 
 # need bootclasspath to point to jdk1.6 rt.jar bootstrap classes
@@ -68,15 +70,7 @@ LOCAL_PROPERTIES_FILE="./build.local.conf"
 
 function clean() {
     echo "cleaning..."
-    rm -fr ${CLASSES}
-    rm -fr ${JAR_ROOT}/h2o_core.jar
-    rm -fr ${JAR_ROOT}/water
-    rm -fr ${JAR_ROOT}/hex
     rm -fr ${OUTDIR}
-    # Also remove prior build system files, for a smooth transition for people 
-    # who try to build without doing a clean wipe
-    rm -fr ${JAR_ROOT}/init
-    rm -fr ${JAR_ROOT}/hexbase_impl.jar
     if [ "$WIPE_TMP" = "true" ]; then
         echo " - wiping tmp..."
         rm -fr /tmp/h2o-temp-*
@@ -111,7 +105,6 @@ function build_initializer() {
 
 function build_jar() {
     JAR_TIME=`date "+%H.%M.%S-%m%d%y"`
-    local JAR_FILE="${OUTDIR}/h2o.jar"
     echo "creating jar file... ${JAR_FILE}"
     # include all libraries
     cd ${JAR_ROOT}
@@ -124,9 +117,9 @@ function build_jar() {
     echo "copying jar file... ${JAR_FILE} to ${OUTDIR}/h2o-${JAR_TIME}.jar"
     cp ${JAR_FILE} ${OUTDIR}/h2o-${JAR_TIME}.jar
 }
-function test_py() {
-    echo "Running junit tests..."
-    python py/junit.py
+function junit() {
+    echo "running JUnit tests..."
+    $JAVA -cp ${JAR_FILE} water.Boot -mainClass water.JUnitRunner
 }
 
 clean
@@ -136,4 +129,4 @@ if [ "$1" = "compile" ]; then exit 0; fi
 build_initializer
 build_jar
 if [ "$1" = "build" ]; then exit 0; fi
-test_py
+junit

ec2

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+java -ea -cp target/h2o.jar water.Boot -mainClass water.sys.EC2 $@

py/cypof.py

@@ -1,28 +1,20 @@
-import h2o, h2o_cmd
-import h2o_browse as h2b
+import h2o
+import h2o_cmd
+import h2o_hosts
+import sys
 import time
-import psutil
 import webbrowser
 #import os, json, unittest, time, shutil, sys
 #sys.path.extend(['.','..','py'])
 
-try:
-    for proc in psutil.process_iter():
-      if proc.name == 'java.exe':
-        proc.kill()
-
-    print 'Building cloud'
-    h2o.clean_sandbox()
-    #h2o.parse_our_args()
-    h2o.build_cloud(4, java_heap_GB=1, capture_output=False, classpath=True)
-    # h2o.nodes = [h2o.ExternalH2O()]
-    print 'KMeans'
-    file = csvPathname = h2o.find_file('smalldata/covtype/covtype.20k.data')
-    h2o_cmd.runKMeans(csvPathname=file, key='covtype', k=7)
-    print 'Web'
-    webbrowser.open("http://localhost:54323/Progress.html?destination_key=covtype.kmeans")
-except KeyboardInterrupt:
-    print 'Interrupted'
-finally:
-    print 'EAT THE BABIES'
-    # h2o.tear_down_cloud()
+h2o.config_json = "testdir_hosts/pytest_config-cypof.json"
+h2o_hosts.build_cloud_with_hosts()
+
+#h2o.build_cloud(4, java_heap_GB=1, capture_output=False, classpath=True)
+
+file = csvPathname = h2o.find_file('smalldata/covtype/covtype.20k.data')
+h2o_cmd.runKMeans(csvPathname=file, key='covtype', k=7)
+webbrowser.open("http://localhost:54323/Progress.html?destination_key=covtype.kmeans")
+
+time.sleep(1)
+h2o.tear_down_cloud()

py/testdir_hosts/pytest_config-cypof.json

@@ -0,0 +1,35 @@
+{
+	"username": "cypof", 
+
+    "use_home_for_ice": true,
+    "inherit_io": true,
+
+    "h2o_per_host": 1,
+    "java_heap_GB": 4,
+
+    "hdfs_name_node": "192.168.1.176",
+    "hdfs_version": "cdh3u5",
+    "use_hdfs": true,
+
+    "ip": [
+        "192.168.1.150", 
+        "192.168.1.151", 
+        "192.168.1.152", 
+        "192.168.1.153", 
+        "192.168.1.154", 
+        "192.168.1.155"
+    ],
+
+    "_ip": [
+        "192.168.1.171",
+        "192.168.1.172",
+        "192.168.1.173",
+        "192.168.1.174",
+        "192.168.1.175",
+        "192.168.1.176",
+        "192.168.1.177",
+        "192.168.1.178",
+        "192.168.1.179",
+        "192.168.1.180"
+    ]
+}

src/main/java/water/Arguments.java

@@ -117,6 +117,16 @@ static public class Req extends Arg {
    */
   public int size() { return commandLineArgs.length;  }
 
+  /**
+   * Index of first flag, or -1 if command line contains only options.
+   */
+  public int firstFlag() {
+    for( int i = 0; i < commandLineArgs.length; i++ )
+      if(commandLineArgs[i].flag)
+        return i;
+    return -1;
+  }
+
   public String get(int i) {  return commandLineArgs[i].val;  }
 
   /**

src/main/java/water/Boot.java

@@ -6,8 +6,7 @@
 import java.net.*;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.Enumeration;
+import java.util.*;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
@@ -112,15 +111,22 @@ else if( (args[i].equals("--ice_root") || args[i].startsWith("-ice_root")) && i
       addInternalJars("jets3t");
     }
 
+    run(args);
+  }
+
+  public static void run(String[] args) throws Exception {
     // Figure out the correct main class to call
     String mainClass = "water.H2O"; // Default mainClass
-    if( args.length >= 2 && args[0].equals("-mainClass") ) {
-      mainClass = args[1];    // Swap out for requested main
-      args = Arrays.copyOfRange(args, 2, args.length);
+    if(args != null) {
+      int index = Arrays.asList(args).indexOf("-mainClass");
+      if( index >= 0 && args.length > index + 1 ) {
+        mainClass = args[index + 1];    // Swap out for requested main
+        args = Arrays.copyOfRange(args, 2, args.length);
+      }
     }
 
     // Call "main"!
-    Class h2oclazz = loadClass(mainClass,true);
+    Class h2oclazz = _init.loadClass(mainClass,true);
     h2oclazz.getMethod("main",String[].class).invoke(null,(Object)args);
   }
 
@@ -219,4 +225,37 @@ private final Class loadClass2( String name ) throws ClassNotFoundException {
     return z;
   }
 
+  // --------------------------------------------------------------------------
+  //
+  // Lists H2O classes
+  //
+  // --------------------------------------------------------------------------
+
+  public static List<String> getClasses() {
+    Log._dontDie = true; // Ignore fatal class load error, e.g. Request
+    ArrayList<String> names = new ArrayList<String>();
+    if(_init._h2oJar != null) {
+      for( Enumeration<ZipEntry> e = (Enumeration) _init._h2oJar.entries(); e.hasMoreElements(); ) {
+        String name = e.nextElement().getName();
+        if( name.endsWith(".class") )
+          names.add(name);
+      }
+    } else
+      findClasses(new File(CLASSES), names);
+
+    for( int i = 0; i < names.size(); i++ ) {
+      String n = names.get(i);
+      names.set(i, n.replace('\\', '/').replace('/', '.').substring(0, n.length() - 6));
+    }
+    return names;
+  }
+  private static final String CLASSES = "target/classes";
+  private static void findClasses(File folder, ArrayList<String> names) {
+    for( File file : folder.listFiles() ) {
+      if( file.isDirectory() )
+        findClasses(file, names);
+      else if( file.getPath().endsWith(".class") )
+        names.add(file.getPath().substring(CLASSES.length() + 1));
+    }
+  }
 }

src/main/java/water/H2O.java

@@ -483,6 +483,9 @@ public static class OptArgs extends Arguments.Opt {
     public int port; // set_cloud_name_and_mcast()
     public String ip; // Named IP4/IP6 address instead of the default
     public String ice_root; // ice root directory
+    public String hdfs; // HDFS backend
+    public String hdfs_version; // version of the filesystem
+    public String hdfs_config; // configuration file of the HDFS
     public String aws_credentials; // properties file for aws credentials
     public String keepice; // Do not delete ice on startup
     public String soft = null; // soft launch for demos
@@ -768,6 +771,17 @@ private static HashSet<H2ONode> parseFlatFile( String fname ) {
     File f = new File(fname);
     if( !f.exists() ) return null; // No flat file
     HashSet<H2ONode> h2os = new HashSet<H2ONode>();
+    List<FlatFileEntry> list = parseFlatFile(f);
+    for(FlatFileEntry entry : list)
+      h2os.add(H2ONode.intern(entry.inet, entry.port+1));// use the UDP port here
+    return h2os;
+  }
+  public static class FlatFileEntry {
+    public InetAddress inet;
+    public int port;
+  }
+  public static List<FlatFileEntry> parseFlatFile( File f ) {
+    List<FlatFileEntry> list = new ArrayList<FlatFileEntry>();
     BufferedReader br = null;
     int port = DEFAULT_PORT;
     try {
@@ -805,11 +819,14 @@ private static HashSet<H2ONode> parseFlatFile( String fname ) {
             Log.die("Invalid port #: "+portStr);
           }
         }
-        h2os.add(H2ONode.intern(inet, port+1));// use the UDP port here
+        FlatFileEntry entry = new FlatFileEntry();
+        entry.inet = inet;
+        entry.port = port;
+        list.add(entry);
       }
     } catch( Exception e ) { Log.die(e.toString()); }
     finally { Closeables.closeQuietly(br); }
-    return h2os;
+    return list;
   }
 
   static void initializePersistence() {

src/main/java/water/Log.java

@@ -87,6 +87,20 @@ public static void unwrap(PrintStream stream, String s) {
       stream.println(s);
   }
 
+  public static void log(File file, PrintStream stream) throws Exception {
+    BufferedReader reader = new BufferedReader(new FileReader(file));
+    try {
+      for( ;; ) {
+        String line = reader.readLine();
+        if( line == null )
+          break;
+        stream.println(line);
+      }
+    } finally {
+      reader.close();
+    }
+  }
+
   private static final class Wrapper extends PrintStream {
     Wrapper(PrintStream parent) {
       super(parent);

src/main/java/water/TypeMap.java

@@ -1,6 +1,5 @@
 package water;
 
-import java.io.File;
 import java.util.*;
 
 public class TypeMap {
@@ -197,31 +196,21 @@ static public Class clazz(int id) {
     return GOLD[id].getClass();
   }
 
+  //
+
   public static void main(String[] args) {
-    Log._dontDie = true; // Ignore class load error, e.g. Request
-    File classes = new File(CLASSES);
     ArrayList<String> list = new ArrayList<String>();
-    findClasses(classes, list);
+    for(String name : Boot.getClasses()) {
+      try {
+        Class c = Class.forName(name);
+        if(Freezable.class.isAssignableFrom(c))
+          list.add(c.getName());
+      } catch(Throwable _) {
+        System.out.println("Skipped: " + name);
+      }
+    }
     Collections.sort(list);
     for(String s : list)
       System.out.println("    \"" + s + "\",");
   }
-  private static final String CLASSES = "target/classes";
-  private static void findClasses(File folder, ArrayList<String> list) {
-    for( File file : folder.listFiles() ) {
-      if( file.isDirectory() )
-        findClasses(file, list);
-      else if( file.getPath().endsWith(".class") ) {
-        String name = file.getPath().substring(CLASSES.length() + 1);
-        name = name.replace('\\', '/').replace('/', '.').replace(".class", "");
-        try {
-          Class c = Class.forName(name);
-          if(Freezable.class.isAssignableFrom(c))
-            list.add(c.getName());
-        } catch(Throwable _) {
-          System.out.println("Skipped: " + name);
-        }
-      }
-    }
-  }
 }