Merge branch 'master' of github.com:0xdata/h2o

src/main/java/water/H2ONode.java

@@ -24,6 +24,7 @@
 public class H2ONode extends Iced implements Comparable {
   public int _unique_idx; // Dense integer index, skipping 0.  NOT cloud-wide unique.
   public long _last_heard_from; // Time in msec since we last heard from this Node
+  public boolean _announcedLostContact;  // True if heartbeat published a no-contact msg
   public volatile HeartBeat _heartbeat;  // My health info.  Changes 1/sec.
   public int _tcp_readers;               // Count of started TCP reader threads
 

src/main/java/water/HeartBeatThread.java

@@ -26,7 +26,7 @@ public HeartBeatThread() {
 
   // Timeout in msec before we decide to not include a Node in the next round
   // of Paxos Cloud Membership voting.
-  static final int TIMEOUT = 60000;
+  static public final int TIMEOUT = 60000;
 
   // Timeout in msec before we decide a Node is suspect, and call for a vote
   // to remove him.  This must be strictly greater than the TIMEOUT.
@@ -113,9 +113,15 @@ public void run() {
 
       // Look for napping Nodes & propose removing from Cloud
       for( H2ONode h2o : cloud._memary ) {
-        if( now - h2o._last_heard_from > SUSPECT ) {  // We suspect this Node has taken a dirt nap
-          Paxos.print("hart: announce suspect node",cloud._memary,h2o.toString());
-          break;
+        long delta = now - h2o._last_heard_from;
+        if( delta > SUSPECT ) {// We suspect this Node has taken a dirt nap
+          if( !h2o._announcedLostContact ) {
+            Paxos.print("hart: announce suspect node",cloud._memary,h2o.toString());
+            h2o._announcedLostContact = true;
+          }
+        } else if( h2o._announcedLostContact ) {
+          Paxos.print("hart: regained contact with node",cloud._memary,h2o.toString());
+          h2o._announcedLostContact = false;
         }
       }
     }

src/main/java/water/api/Cloud.java

@@ -46,6 +46,7 @@ public Cloud() {
       node.addProperty(FJ_QUEUE_LO, (int)hb._fjqueue_lo);
       node.addProperty(RPCS, (int)hb._rpcs);
       node.addProperty(TCPS_ACTIVE, (int) hb._tcps_active);
+      node.addProperty(LAST_CONTACT,h2o._last_heard_from);
       nodes.add(node);
     }
     response.add(NODES,nodes);
@@ -54,7 +55,9 @@ public Cloud() {
     Response r = Response.done(response);
     r.setBuilder(CONSENSUS, new BooleanStringBuilder("","Voting new members"));
     r.setBuilder(LOCKED, new BooleanStringBuilder("Locked","Accepting new members"));
+    r.setBuilder(NODES, new MyAryBuilder());
     r.setBuilder(NODES+"."+NAME, new NodeCellBuilder());
+    r.setBuilder(NODES+"."+LAST_CONTACT, new LastContactBuilder());
     return r;
   }
 
@@ -63,12 +66,32 @@ public static String pos_neg(double d) {
   }
 
   // Just the Node as a link
-  public class NodeCellBuilder extends ArrayRowElementBuilder {
+  private static class NodeCellBuilder extends ArrayRowElementBuilder {
     @Override public String elementToString(JsonElement element, String contextName) {
       String str = element.getAsString();
       if( str.equals(H2O.SELF.toString()) )
         return "<a href='StoreView.html'>"+str+"</a>";
       return "<a href='Remote.html?Node="+str+"'>"+str+"</a>";
     }
   }
+  // Highlight sick nodes
+  private static class MyAryBuilder extends ArrayBuilder {
+    static ArrayRowBuilder MY_ARRAY_ROW = new MyRowBuilder();
+    @Override public Builder defaultBuilder(JsonElement element) { return MY_ARRAY_ROW; }
+  }
+  private static class MyRowBuilder extends ArrayRowBuilder {
+    @Override public String header(JsonObject object, String objectName) {
+      long then = object.getAsJsonPrimitive(LAST_CONTACT).getAsLong();
+      long now = System.currentTimeMillis();
+      return ((now-then) >= HeartBeatThread.TIMEOUT) ? "\n<tr class=\"error\">" : "\n<tr>";
+    }
+  }
+  // Last-heard-from time pretty-printing
+  private static class LastContactBuilder extends ArrayRowElementBuilder {
+    @Override public String elementToString(JsonElement element, String contextName) {
+      long then = element.getAsLong();
+      long now = System.currentTimeMillis();
+      return (now-then >= 2*1000) ? ""+((now-then)/1000)+" secs ago" : "now";
+    }
+  }
 }

src/main/java/water/api/Constants.java

@@ -85,6 +85,7 @@ public static class Schemes {
   public static final String K                     = "k";
   public static final String KEY                   = "key";
   public static final String KEYS                  = "keys";
+  public static final String LAST_CONTACT          = "last_contact";
   public static final String LAMBDA                = "lambda";
   public static final String LIMIT                 = "limit";
   public static final String LINK                  = "link";

src/main/java/water/parser/Enum.java

@@ -22,7 +22,7 @@
  */
 public final class Enum extends Iced {
 
-  public static final int MAX_ENUM_SIZE = 11000;
+  public static final int MAX_ENUM_SIZE = 65000;
 
   volatile NonBlockingHashMap<ValueString, Integer> _map;