Much more extensive JSON support

Nested structures will print out nested JSON.
Arrays of Iced will JSON.
Nested structures will auto-doc-gen all nested classes.
Fix bug with chunks that got written-to, not cleaning up state.
Frame reports byteSize for whole frame.
Include Vec._mean in the rollups.

src/main/java/water/AutoBuffer.java

@@ -984,4 +984,28 @@ public AutoBuffer putJSONAStr(String name, String[] fs) {
     }
     return put1(']');
   }
+
+  public AutoBuffer putJSON( Iced ice ) {
+    return ice == null ? putNULL() : ice.writeJSON(this);
+  }
+  public AutoBuffer putJSONA( Iced fs[] ) {
+    if( fs == null ) return putNULL();
+    put1('[');
+    for( int i=0; i<fs.length; i++ ) {
+      if( i>0 ) put1(',');
+      putJSON(fs[i]);
+    }
+    return put1(']');
+  }
+
+  public AutoBuffer putEnumJSON( Enum e ) {
+    return e==null ? putNULL() : put1('"').putStr2(e.toString()).put1('"');
+  }
+
+  public AutoBuffer putJSON  ( String name, Iced f   ) { return putJSONStr(name).put1(':').putJSON (f); }
+  public AutoBuffer putJSONA ( String name, Iced f[] ) { return putJSONStr(name).put1(':').putJSONA(f); }
+  public AutoBuffer putJSON8d( String name, double d ) { return putJSONStr(name).put1(':').putStr2(Double .toString(d)); }
+  public AutoBuffer putJSON8 ( String name, long l   ) { return putJSONStr(name).put1(':').putStr2(Long   .toString(l)); }
+  public AutoBuffer putJSON4 ( String name, int i    ) { return putJSONStr(name).put1(':').putStr2(Integer.toString(i)); }
+  public AutoBuffer putEnumJSON( String name, Enum e ) { return putJSONStr(name).put1(':').putEnumJSON(e); }
 }

src/main/java/water/DTask.java

@@ -50,5 +50,7 @@ private RuntimeException barf() {
   @Override public <F extends Freezable> F read(AutoBuffer bb) { throw barf(); }
   @Override public <F extends Freezable> F newInstance() { throw barf(); }
   @Override public int frozenType() { throw barf(); }
+  @Override public AutoBuffer writeJSONFields(AutoBuffer bb) { throw barf(); }
+  @Override public water.api.DocGen.FieldDoc[] toDocField() { return null; }
   public void copyOver(T that) { throw barf(); }
 }

src/main/java/water/Freezable.java

@@ -12,4 +12,9 @@ public interface Freezable {
   public <T extends Freezable> T newInstance();
   /** Return the cluster-wide-unique 2-byte type ID for instances of this class */
   public int frozenType();
+  /** Serialize the 'this' object into the AutoBuffer, returning the AutoBuffer. 
+      Output is legal JSON. */
+  public AutoBuffer writeJSONFields(AutoBuffer bb);
+  /** Reflective list of fields */
+  public water.api.DocGen.FieldDoc[] toDocField();
 }

src/main/java/water/Iced.java

@@ -14,6 +14,9 @@ private RuntimeException barf() {
   @Override public <T extends Freezable> T read(AutoBuffer bb) { throw barf(); }
   @Override public <T extends Freezable> T newInstance() { throw barf(); }
   @Override public int frozenType() { throw barf(); }
+  @Override public AutoBuffer writeJSONFields(AutoBuffer bb) { throw barf(); }
+  public final AutoBuffer writeJSON(AutoBuffer bb) { return writeJSONFields(bb.put1('{')).put1('}'); }
+  @Override public water.api.DocGen.FieldDoc[] toDocField() { return null; }
 
   public Iced init( Key k ) { return this; }
 }

src/main/java/water/Weaver.java

@@ -119,7 +119,7 @@ private static boolean hasExisting( String methname, String methsig, CtBehavior
   // water.DTask.  Add any missing serialization methods.
   CtClass addSerializationMethods( CtClass cc ) throws CannotCompileException, NotFoundException {
     if( cc.subclassOf(_enum) ) exposeRawEnumArray(cc);
-    if( cc.subclassOf(_api ) ) ensureAPImethods(cc);
+    if( cc.subclassOf(_iced) || cc.subclassOf(_api ) ) ensureAPImethods(cc);
     if( cc.subclassOf(_iced) ||
         cc.subclassOf(_dtask) ) {
       cc.setModifiers(javassist.Modifier.setPublic(cc.getModifiers()));
@@ -222,13 +222,13 @@ private void ensureAPImethods(CtClass cc) throws NotFoundException, CannotCompil
     // ---
     // Auto-gen JSON output to AutoBuffers
     make_body(cc,ctfs,false,
-              "public water.AutoBuffer writeJSON(water.AutoBuffer ab) {\n  ab.put1('{');\n",
-              "  super.writeJSON(ab);\n",
+              "public water.AutoBuffer writeJSONFields(water.AutoBuffer ab) {\n",
+              "  super.writeJSONFields(ab);\n",
+              "  ab.putJSON%z(\"%s\",%s)",
+              "  ab.putEnumJSON(\"%s\",%s)",
               "  ab.putJSON%z(\"%s\",%s)",
-              "  ab.putEnumJSON(%s)",
-              "  ab.putJSON%z(%s)",
               ".put1(',');\n",
-              ";\n  return ab.put1('}');\n}",
+              ";\n  return ab;\n}",
               new FieldFilter() {
                 boolean filter(CtField ctf) throws NotFoundException {return !ctf.getType().subclassOf(_arg); }
               });

src/main/java/water/api/DocGen.java

@@ -130,13 +130,7 @@ public String genHelp(Request R) {
     listTail(sb);
 
     section(sb,"Output JSON elements");
-    listHead(sb);
-    for( FieldDoc doc : docs )
-      if( doc.isJSON() )
-        listBullet(sb,
-                   bold(doc._name)+", a "+doc._clazz.getSimpleName(),
-                   doc._help+doc.version(),0);
-    listTail(sb);
+    listJSONFields(sb,docs);
 
     section(sb,"HTTP response codes");
     paragraph(sb,"200 OK");
@@ -151,17 +145,40 @@ public String genHelp(Request R) {
       paragraph(sb,serve(name,s));
     }
 
-    section(sb,"Error Example");
     String f[] = R.DocExampleFail();
-    paraHead(sb);
-    url(sb,name,f);
-    paraTail(sb);
-    paragraph(sb,serve(name,f));
+    if( f != null ) {
+      section(sb,"Error Example");
+      paraHead(sb);
+      url(sb,name,f);
+      paraTail(sb);
+      paragraph(sb,serve(name,f));
+    }
 
     bodyTail(sb);
     return sb.toString();
   }
 
+  private void listJSONFields( StringBuilder sb, FieldDoc[] docs ) {
+    listHead(sb);
+    for( FieldDoc doc : docs )
+      if( doc.isJSON() ) {
+        listBullet(sb,
+                   bold(doc._name)+", a "+doc._clazz.getSimpleName(),
+                   doc._help+doc.version(),0);
+        Class c = doc._clazz.getComponentType();
+        if( c==null ) c = doc._clazz;
+        if( Iced.class.isAssignableFrom(c) ) {
+          try {
+            FieldDoc[] nested = ((Iced)c.newInstance()).toDocField();
+            listJSONFields(sb,nested);
+          } 
+          catch( InstantiationException ie ) { water.util.Log.errRTExcept(ie); }
+          catch( IllegalAccessException ie ) { water.util.Log.errRTExcept(ie); }
+        }
+      }
+    listTail(sb);
+  }
+
   private static StringBuilder url( StringBuilder sb, String name, String[] parms ) {
     sb.append("curl -s ").append(name).append(".json");
     boolean first = true;

src/main/java/water/api/Inspect2.java

@@ -15,74 +15,122 @@ public class Inspect2 extends Request {
   @Weave(help="An existing H2O Frame key.")
   final FrameKey src_key = new FrameKey("src_key");
 
+  @Weave(help="Number of data rows.") long numRows;
+  @Weave(help="Number of data columns.") int numCols;
+  @Weave(help="byte size in memory.") long byteSize;
+
+  // An internal JSON-output-only class
+  private static class ColSummary extends Iced {
+    static final int API_WEAVER=1; // This file has auto-gen'd doc & json fields
+    static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.
+    public ColSummary( String name, Vec vec ) {
+      this.name = name;  
+      this.min  = vec.min();
+      this.max  = vec.max();
+      this.mean = vec.mean();
+      this.NAcnt= vec.NAcnt();
+      this.type = vec.dtype();
+    }
+    @Weave(help="Label."           ) final String name;    
+    @Weave(help="min."             ) final double min;
+    @Weave(help="max."             ) final double max;
+    @Weave(help="mean."            ) final double mean;
+    @Weave(help="Missing elements.") final long   NAcnt;
+    @Weave(help="Data type, one of I=Integer, F=Float, S=String, NA=all rows missing.")
+    final Vec.DType type;
+  }
+
+  @Weave(help="Array of Column Summaries.")
+  ColSummary cols[];
+
+
+  // Called from some other page, to redirect that other page to this page.
   public static Response redirect(Request req, String src_key) {
     return new Response(Response.Status.redirect, req, -1, -1, "Inspect2", "src_key", src_key );
   }
 
+  // Just validate the frame, and fill in the summary bits
   @Override protected Response serve() {
     Frame fr = DKV.get(src_key.value()).get();
     if( fr == null ) return RequestServer._http404.serve();
+    numRows = fr.numRows();
+    numCols = fr.numCols();
+    byteSize = fr.byteSize();
+    cols = new ColSummary[numCols];
+    for( int i=0; i<cols.length; i++ )
+      cols[i] = new ColSummary(fr._names[i],fr._vecs[i]);
+
     return new Response(Response.Status.done, this, -1, -1, null);
   }
 
   @Override public boolean toHTML( StringBuilder sb ) {
     Key skey = src_key.value();
-    Frame fr = DKV.get(skey).get();
-    final int numCols = fr.numCols();
 
     DocGen.HTML.title(sb,skey.toString());
+    DocGen.HTML.section(sb,""+numCols+" columns, "+numRows+" rows, "+PrettyPrint.bytes(byteSize));
     DocGen.HTML.arrayHead(sb);
     // Column labels
     sb.append("<tr class='warning'>");
     sb.append("<td>").append("Row").append("</td>");
-    for( int i=0; i<numCols; i++ ) 
-      sb.append("<td><b>").append(fr._names[i]).append("</b></td>");
+    for( int i=0; i<cols.length; i++ ) 
+      sb.append("<td><b>").append(cols[i].name).append("</b></td>");
     sb.append("</tr>");
 
     //sb.append("<tr class='warning'>");
     //sb.append("<td>").append("Bytes").append("</td>");
-    //for( int i=0; i<numCols; i++ ) 
+    //for( int i=0; i<cols.length; i++ ) 
     //  sb.append("<td>").append(PrettyPrint.bytes(fr._vecs[i].byteSize())).append("</td>");
     //sb.append("</tr>");
 
     sb.append("<tr class='warning'>");
     sb.append("<td>").append("Min").append("</td>");
-    for( int i=0; i<numCols; i++ )
-      sb.append("<td>").append(x3(fr._vecs[i].min())).append("</td>");
+    for( int i=0; i<cols.length; i++ )
+      sb.append("<td>").append(x3(cols[i].min)).append("</td>");
     sb.append("</tr>");
 
     sb.append("<tr class='warning'>");
     sb.append("<td>").append("Max").append("</td>");
-    for( int i=0; i<numCols; i++ ) 
-      sb.append("<td>").append(x3(fr._vecs[i].max())).append("</td>");
+    for( int i=0; i<cols.length; i++ ) 
+      sb.append("<td>").append(x3(cols[i].max)).append("</td>");
+    sb.append("</tr>");
+
+    sb.append("<tr class='warning'>");
+    sb.append("<td>").append("Mean").append("</td>");
+    for( int i=0; i<cols.length; i++ ) 
+      sb.append("<td>").append(String.format("%5.3f",cols[i].mean)).append("</td>");
+    sb.append("</tr>");
+
+    sb.append("<tr class='warning'>");
+    sb.append("<td>").append("Type").append("</td>");
+    for( int i=0; i<cols.length; i++ ) 
+      sb.append("<td>").append(cols[i].type).append("</td>");
     sb.append("</tr>");
 
     boolean hasNAs = false;
-    long nas[] = new long[numCols];
-    for( int i=0; i<numCols; i++ ) 
-      if( (nas[i]=fr._vecs[i].NAcnt()) > 0 ) hasNAs = true;
+    for( int i=0; i<cols.length; i++ ) 
+      if( cols[i].NAcnt > 0 ) hasNAs = true;
 
     if( hasNAs ) {
       sb.append("<tr class='warning'>");
       sb.append("<td>").append("Missing").append("</td>");
-      for( int i=0; i<numCols; i++ ) 
-        sb.append("<td>").append(nas[i] > 0 ? Long.toString(nas[i]) : "").append("</td>");
+      for( int i=0; i<cols.length; i++ ) 
+        sb.append("<td>").append(cols[i].NAcnt > 0 ? Long.toString(cols[i].NAcnt) : "").append("</td>");
       sb.append("</tr>");
     }
 
     // First N rows
-    int N = (int)Math.min(100,fr.numRows());
+    Frame fr = DKV.get(skey).get();
+    int N = (int)Math.min(100,numRows);
     for( int j=0; j<N; j++ ) {
       sb.append("<tr>");
       sb.append("<td>").append(j).append("</td>");
-      for( int i=0; i<numCols; i++ ) 
+      for( int i=0; i<cols.length; i++ ) 
         sb.append("<td>").append(x1(fr._vecs[i],j)).append("</td>");
       sb.append("</tr>");
     }
 
     DocGen.HTML.arrayTail(sb);
 
-    //throw H2O.unimpl();
     return true;
   }
 
@@ -95,15 +143,8 @@ private String x1( Vec v, int row ) {
     case F: {
       Chunk c = v.elem2BV(0);
       Class Cc = c.getClass();
-      if(        Cc == null ) {
-      } else if( Cc == C1SChunk.class ) {
-        return x2(v,row,((C1SChunk)c)._scale);
-      } else if( Cc == C2SChunk.class ) {
-        return x2(v,row,((C2SChunk)c)._scale);
-      } else {
-        return Double.toString(v.at (row));
-        //throw H2O.unimpl();
-      }
+      if( Cc == C1SChunk.class ) return x2(v,row,((C1SChunk)c)._scale);
+      if( Cc == C2SChunk.class ) return x2(v,row,((C2SChunk)c)._scale);
       return Double.toString(v.at (row));
     }
     case S:

src/main/java/water/api/Request.java

@@ -208,7 +208,8 @@ public static JsonObject execSync(Request request) {
   private RuntimeException barf() {
     return new RuntimeException(getClass().toString()+" should be automatically overridden in the subclass by the Weaver");
   }
-  public AutoBuffer writeJSON( AutoBuffer ab ) { throw barf(); }
+  public AutoBuffer writeJSONFields( AutoBuffer ab ) { throw barf(); }
+  public final AutoBuffer writeJSON( AutoBuffer ab ) { return writeJSONFields(ab.put1('{')).put1('}'); }
   public boolean toHTML( StringBuilder sb ) { return false; }
   public DocGen.FieldDoc[] toDocField() { return null; }
   public String toDocGET() { return null; }

src/main/java/water/fvec/AppendableVec.java

@@ -26,7 +26,7 @@ public class AppendableVec extends Vec {
   long _totalCnt;
 
   AppendableVec( Key key) {
-    super(key, null,false,Double.MAX_VALUE,Double.MIN_VALUE,0);
+    super(key, null,false,0);
     _espc = new long[4];
     _chunkTypes = new byte[4];
   }
@@ -42,11 +42,7 @@ synchronized void closeChunk( NewChunk chk) {
     }
     _espc[cidx] = chk._len;
     _chunkTypes[cidx] = chk.type();
- //   _homes[cidx] = H2O.SELF.index();
     _hasFloat |= chk.hasFloat();
-    // Roll-up totals for each chunk as it closes
-    if( chk._min < _min ) _min = chk._min;
-    if( chk._max > _max ) _max = chk._max;
     _missingCnt += chk._naCnt;
     _strCnt += chk._strCnt;
     _totalCnt += chk._len;
@@ -114,7 +110,7 @@ public Vec close(Futures fs) {
     }
     espc[nchunk]=x;             // Total element count in last
     // Replacement plain Vec for AppendableVec.
-    Vec vec = new Vec(_key, espc,!_hasFloat,_min,_max,_missingCnt);
+    Vec vec = new Vec(_key, espc,!_hasFloat,_missingCnt);
     vec._dtype = dtype();
     DKV.put(_key,vec,fs);       // Inject the header
     return vec;

src/main/java/water/fvec/ByteVec.java

@@ -8,7 +8,7 @@
 // A vector of plain Bytes.
 public class ByteVec extends Vec {
 
-  ByteVec( Key key, long espc[] ) { super(key,espc,true,Double.NaN,Double.NaN, 0); }
+  ByteVec( Key key, long espc[] ) { super(key,espc,true,0); }
 
   public C1NChunk elem2BV( int cidx ) { return (C1NChunk)super.elem2BV(cidx); }
 

src/main/java/water/fvec/Chunk.java

@@ -70,6 +70,7 @@ public final long set80(int idx, long l) {
       assert !(this instanceof NewChunk) : "Cannot direct-write into a NewChunk, only append";
       _vec.startWriting();      // One-shot writing-init
       _chk = clone();           // Flag this chunk as having been written into
+      _chk._chk = _chk;         // Clone has NOT been written into
     }
     if( _chk.set8_impl(idx,l) ) return l;
     // Must inflate the chunk
@@ -93,6 +94,7 @@ public final double set80(int idx, double d) {
       assert !(this instanceof NewChunk) : "Cannot direct-write into a NewChunk, only append";
       _vec.startWriting();      // One-shot writing-init
       _chk = clone();           // Flag this chunk as having been written into
+      _chk._chk = _chk;         // Clone has NOT been written into
     }
     if( _chk.set8_impl(idx,d) ) return d;
     // Must inflate the chunk

src/main/java/water/fvec/Frame.java

@@ -113,6 +113,14 @@ public void remove(Futures fs){
   public void remove() {
     remove(new Futures());
   }
+
+  public long byteSize() {
+    long sum=0;
+    for( int i=0; i<_vecs.length; i++ )
+      sum += _vecs[i].byteSize();
+    return sum;
+  }
+
   @Override public String toString() {
     // Across
     String s="{"+_names[0];