Samples: fixed new API, added debug plot to k-means

Author: cypof

.gitignore

@@ -24,4 +24,5 @@ syn_datasets
 target/
 .settings/*
 cypof*
-typemap
+typemap
+/libpeerconnection.log

src/main/java/hex/Plot.java

@@ -46,14 +46,21 @@ static class Pixels extends MRTask {
         Column cY = va._cols[_cols[1]];
         double x = va.datad(bits, row, cX);
         double y = va.datad(bits, row, cY);
-        int iX = scale(x, cX, _width);;
+        int iX = scale(x, cX, _width);
         int iY = scale(y, cY, _height);
         int value = _pixels[iY * _width + iX] & 0xff;
         value = value == 0xff ? value : value + 1;
-        _pixels[iY * _width + iX] = (byte) value;
+        dot(iX, iY, value);
       }
     }
 
+    private void dot(int iX, int iY, int value) {
+      int r = 2;
+      for( int y = Math.max(0, iY - r); y < Math.min(_height, iY + r); y++ )
+        for( int x = Math.max(0, iX - r); x < Math.min(_width, iX + r); x++ )
+          _pixels[y * _width + x] = (byte) value;
+    }
+
     @Override public void reduce(DRemoteTask rt) {
       Pixels task = (Pixels) rt;
 

src/main/java/water/api/Plot.java

@@ -66,7 +66,7 @@ public NanoHTTPD.Response serve(NanoHTTPD server, Properties args, RequestType t
             pixel[0] = 255;
             pixel[1] = 0;
             pixel[2] = 0;
-            int r = 2;
+            int r = 3;
             for( int y = Math.max(0, y0 - r); y < Math.min(height, y0 + r); y++ )
               for( int x = Math.max(0, x0 - r); x < Math.min(width, x0 + r); x++ )
                 raster.setPixel(x, y, pixel);

src/main/java/water/fvec/C4Chunk.java

@@ -8,7 +8,7 @@ public class C4Chunk extends Chunk {
   @Override public long   get ( int    i ) { return UDP.get4(_mem,i<<2); }
   @Override public double getd( int    i ) { return UDP.get4(_mem,i<<2); }
   @Override void   append2 ( long l, int exp ) { throw H2O.fail(); }
-  @Override public AutoBuffer write(AutoBuffer bb) { throw H2O.fail(); }
+  @Override public AutoBuffer write(AutoBuffer bb) { return bb.putA1(_mem,_mem.length); }
   @Override public C4Chunk read(AutoBuffer bb) {
     _mem = bb.bufClose();
     _start = -1;

src/main/java/water/fvec/C4FChunk.java

@@ -8,7 +8,7 @@ public class C4FChunk extends Chunk {
   @Override public long   get ( int    i ) { return (long)UDP.get4f(_mem,i<<2); }
   @Override public double getd( int    i ) { return       UDP.get4f(_mem,i<<2); }
   @Override void   append2 ( long l, int exp ) { throw H2O.fail(); }
-  @Override public AutoBuffer write(AutoBuffer bb) { throw H2O.fail(); }
+  @Override public AutoBuffer write(AutoBuffer bb) { return bb.putA1(_mem,_mem.length); }
   @Override public C4FChunk read(AutoBuffer bb) {
     _mem = bb.bufClose();
     _start = -1;

src/main/java/water/fvec/C8Chunk.java

@@ -8,7 +8,7 @@ public class C8Chunk extends Chunk {
   @Override public long   get ( int    i ) { return UDP.get8(_mem,i<<3); }
   @Override public double getd( int    i ) { return UDP.get8(_mem,i<<3); }
   @Override void   append2 ( long l, int exp ) { throw H2O.fail(); }
-  @Override public AutoBuffer write(AutoBuffer bb) { throw H2O.fail(); }
+  @Override public AutoBuffer write(AutoBuffer bb) { return bb.putA1(_mem,_mem.length); }
   @Override public C8Chunk read(AutoBuffer bb) {
     _mem = bb.bufClose();
     _start = -1;

src/main/java/water/fvec/C8DChunk.java

@@ -8,7 +8,7 @@ public class C8DChunk extends Chunk {
   @Override public long   get ( int    i ) { return (long)UDP.get8d(_mem,i<<3); }
   @Override public double getd( int    i ) { return       UDP.get8d(_mem,i<<3); }
   @Override void   append2 ( long l, int exp ) { throw H2O.fail(); }
-  @Override public AutoBuffer write(AutoBuffer bb) { throw H2O.fail(); }
+  @Override public AutoBuffer write(AutoBuffer bb) { return bb.putA1(_mem,_mem.length); }
   @Override public C8DChunk read(AutoBuffer bb) {
     _mem = bb.bufClose();
     _start = -1;

src/main/java/water/fvec/Vec.java

@@ -106,7 +106,7 @@ Chunk nextBV( Chunk bv ) {
 
   // Fetch element the slow way
   public long get( long i ) { return elem2BV(elem2ChunkIdx(i)).at(i); }
-  public double getd( long i ) { throw H2O.unimpl(); }
+  public double getd( long i ) { return elem2BV(elem2ChunkIdx(i)).atd(i); }
 
   // [#elems, min/mean/max]
   @Override public String toString() {

src/samples/h2o/samples/Part05_KMeans.java

@@ -1,5 +1,7 @@
 package h2o.samples;
 
+import hex.KMeansModel;
+
 import java.text.DecimalFormat;
 import java.util.Random;
 
@@ -20,26 +22,28 @@ public static void main(String[] args) throws Exception {
       // Starts a one node cluster
       H2O.main(args);
 
-      // Loads, unzip and parse a file. Data is distributed to other nodes in a round-robin way.
-      Key key = TestUtil.loadAndParseFile("test", "smalldata/gaussian/sdss174052.csv.gz");
+      // Load and parse a file. Data is distributed to other nodes in a round-robin way
+      Key key = TestUtil.loadAndParseFile("test.hex", "lib/resources/datasets/gaussian.csv");
 
       // ValueArray is a distributed 2D array
       ValueArray va = DKV.get(key).get();
 
-      // Which columns to use, e.g. skip first for this dataset
-      int[] cols = new int[va._cols.length - 1];
+      // Which columns to use, use all in this case
+      int[] cols = new int[va._cols.length];
       for( int i = 0; i < cols.length; i++ )
-        cols[i] = i + 1;
+        cols[i] = i;
 
       // Create k clusters as arrays of doubles
-      int k = 3;
+      int k = 7;
       double[][] clusters = new double[k][va._cols.length];
 
-      // Initialize first cluster to random row
+      // Initialize clusters to random rows
       Random rand = new Random();
-      long row = Math.max(0, (long) (rand.nextDouble() * va._numrows) - 1);
-      for( int i = 0; i < cols.length; i++ )
-        clusters[0][i] = va.datad(row, cols[i]);
+      for( int cluster = 0; cluster < clusters.length; cluster++ ) {
+        long row = Math.max(0, (long) (rand.nextDouble() * va._numrows) - 1);
+        for( int i = 0; i < cols.length; i++ )
+          clusters[cluster][i] = va.datad(row, cols[i]);
+      }
 
       // Iterate over the dataset and show error for each step
       for( int i = 0; i < 20; i++ ) {
@@ -68,6 +72,13 @@ public static void main(String[] args) throws Exception {
           System.out.print(df.format(clusters[cluster][column]) + ", ");
         System.out.println("");
       }
+
+      // Experimental: debug plot
+      KMeansModel model = new KMeansModel(Key.make("test.kmeans"), cols, key);
+      model._clusters = clusters;
+      UKV.put(model._selfKey, model);
+      System.out.println("Plot:");
+      System.out.println("http://127.0.0.1:54321/Plot.png?source_key=test.hex&cols=0%2C1&clusters=test.kmeans");
     }
   }
 

src/samples/h2o/samples/Part05_KMeansNewAPI.java

@@ -22,22 +22,24 @@ public static class UserMain {
     public static void main(String[] args) throws Exception {
       H2O.main(args);
 
-      // Parse file
-      Key file = NFSFileVec.make(new File("smalldata/gaussian/sdss174052.csv.gz"));
+      // Load and parse a file. Data is distributed to other nodes in a round-robin way
+      Key file = NFSFileVec.make(new File("lib/resources/datasets/gaussian.csv"));
       Frame frame = ParseDataset2.parse(Key.make("test"), new Key[] { file });
 
-      // Create a frame with only the columns to use, e.g. skip first for this dataset
+      // Optionally create a frame with less columns, e.g. skip first
       frame = new Frame(null, Utils.remove(frame._names, 0), Utils.remove(frame._vecs, 0));
 
       // Create k clusters as arrays of doubles
-      int k = 3;
+      int k = 7;
       double[][] clusters = new double[k][frame.vecs().length];
 
       // Initialize first cluster to random row
       Random rand = new Random();
-      long row = Math.max(0, (long) (rand.nextDouble() * frame.vecs().length) - 1);
-      for( int i = 0; i < frame.vecs().length; i++ )
-        clusters[0][i] = frame.vecs()[i].getd(row);
+      for( int cluster = 0; cluster < clusters.length; cluster++ ) {
+        long row = Math.max(0, (long) (rand.nextDouble() * frame.vecs().length) - 1);
+        for( int i = 0; i < frame.vecs().length; i++ )
+          clusters[cluster][i] = frame.vecs()[i].getd(row);
+      }
 
       // Iterate over the dataset and show error for each step
       for( int i = 0; i < 10; i++ ) {
@@ -56,9 +58,8 @@ public static void main(String[] args) throws Exception {
         System.out.println("Error is " + task._error);
       }
 
-      DecimalFormat df = new DecimalFormat("#.00");
-
       System.out.println("Clusters:");
+      DecimalFormat df = new DecimalFormat("#.00");
       for( int c = 0; c < clusters.length; c++ ) {
         for( int v = 0; v < frame.vecs().length; v++ )
           System.out.print(df.format(clusters[c][v]) + ", ");

src/test/java/water/Sandbox.java

@@ -1,5 +1,8 @@
 package water;
 
+import hex.KMeans;
+import hex.KMeansModel;
+
 import java.io.File;
 
 import org.apache.commons.lang.ArrayUtils;
@@ -14,7 +17,7 @@ public static void main(String[] args) throws Exception {
 
     //File f = new File("py/testdir_multi_jvm/syn_datasets/syn_spheres100.csv");
     //File f = new File("smalldata/covtype/covtype.20k.data");
-    File f = new File("smalldata/allstate/claim_prediction_train_set_10000.csv.gz");
+    File f = new File("lib/resources/datasets/gaussian.csv");
     //File f = new File("smalldata/airlines/allyears2k.zip");
     // // File f = new File("../../aaaa/datasets/millionx7_logreg.data.gz");
     // // File f = new File("smalldata/test/rmodels/iris_x-iris-1-4_y-species_ntree-500.rdata");
@@ -30,17 +33,22 @@ public static void main(String[] args) throws Exception {
     // double[][] array = KMeansTest.gauss(columns, 10000, goals);
     // ValueArray va = TestUtil.va_maker(key, (Object[]) array);
 
-//    Key km = Key.make("test.kmeans");
-//    int[] cols = new int[va._cols.length];
-//    for( int i = 0; i < cols.length; i++ )
-//      cols[i] = i;
-//    for( int i = 0; i < 100; i++ ) {
-//      KMeans job = KMeans.start(km, va, 5, 1e-3, 0, i, false, cols);
-//      KMeansModel m = job.get();
-//      System.out.println(m._error);
-//    }
+    Key km = Key.make("test.kmeans");
+    int[] cols = new int[va._cols.length];
+    for( int i = 0; i < cols.length; i++ )
+      cols[i] = i;
+    for( int i = 0; i < 1; i++ ) {
+      KMeans job = KMeans.start(km, va, 7, 1e-3, 0, i, false, cols);
+      KMeansModel m = job.get();
+      System.out.println(m._error);
+    }
 
-//    String u = "/Plot.png?source_key=test.hex&cols=0%2C1&clusters=test.kmeans";
+//    String s = "";
+//    for( int i = 0; i < cols.length; i++ ) {
+//      s += s.length() != 0 ? "%2C" : "";
+//      s += cols[i];
+//    }
+//    String u = "/Plot.png?source_key=test.hex&cols=" + s + "&clusters=test.kmeans";
 //    Desktop.getDesktop().browse(new URI("http://localhost:54321" + u));
   }