HEX-600, HEX-596 & jython scavenging

.classpath

@@ -7,7 +7,7 @@
 	<classpathentry kind="lib" path="lib/apache/commons-lang-2.4.jar" sourcepath="lib/apache/commons-lang-2.4-sources.jar"/>
 	<classpathentry kind="lib" path="lib/apache/commons-logging-1.1.1.jar" sourcepath="lib/apache/commons-logging-1.1.1-src.zip"/>
 	<classpathentry kind="lib" path="lib/apache/httpclient-4.1.1.jar"/>
-	<classpathentry kind="lib" path="lib/apache/httpcore-4.1.jar"/>
+	<classpathentry kind="lib" path="lib/apache/httpcore-4.1.jar" sourcepath="lib/apache/httpcore-4.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/junit/junit-4.11.jar" sourcepath="lib/junit/junit-4.11-sources.jar"/>
 	<classpathentry kind="lib" path="lib/apache/guava-12.0.1.jar" sourcepath="lib/apache/guava-12.0.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/gson/gson-2.2.2.jar" sourcepath="lib/gson/gson-2.2.2-sources.jar"/>

.settings/org.eclipse.jdt.core.prefs

@@ -152,7 +152,7 @@ org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
 org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
 org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
 org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=140
+org.eclipse.jdt.core.formatter.comment.line_length=100
 org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
 org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
 org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
@@ -360,7 +360,7 @@ org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
 org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
 org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
 org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=140
+org.eclipse.jdt.core.formatter.lineSplit=120
 org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
 org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
 org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0

build.sh

@@ -35,7 +35,7 @@ DEPENDENCIES="${JAR_ROOT}/jama/*${SEP}${JAR_ROOT}/apache/*${SEP}${JAR_ROOT}/juni
 DEFAULT_HADOOP_VERSION="1.0.0"
 OUTDIR="target"
 
-JAVAC=`which javac`
+JAVAC=`which javac`||echo 'Missing javac, please install jdk'
 
 # need bootclasspath to point to jdk1.6 rt.jar bootstrap classes
 # extdirs can also be passed as -extdirs 

src/main/java/water/H2O.java

@@ -14,6 +14,8 @@
 import water.store.s3.PersistS3;
 import water.util.Utils;
 
+import com.amazonaws.auth.PropertiesCredentials;
+import com.google.common.base.Objects;
 import com.google.common.io.Closeables;
 
 /**
@@ -481,9 +483,6 @@ public static class OptArgs extends Arguments.Opt {
     public int port; // set_cloud_name_and_mcast()
     public String ip; // Named IP4/IP6 address instead of the default
     public String ice_root; // ice root directory
-    public String hdfs; // HDFS backend
-    public String hdfs_version; // version of the filesystem
-    public String hdfs_config; // configuration file of the HDFS
     public String aws_credentials; // properties file for aws credentials
     public String keepice; // Do not delete ice on startup
     public String soft = null; // soft launch for demos
@@ -523,6 +522,13 @@ private static void initializeExpressionEvaluation() {
     Function.initializeCommonFunctions();
   }
 
+  // Default location of the AWS credentials file
+  private static final String DEFAULT_CREDENTIALS_LOCATION = "AwsCredentials.properties";
+  public static PropertiesCredentials getAWSCredentials() throws IOException {
+    File credentials = new File(Objects.firstNonNull(OPT_ARGS.aws_credentials, DEFAULT_CREDENTIALS_LOCATION));
+    return new PropertiesCredentials(credentials);
+  }
+
   /** Starts the local k-v store.
 * Initializes the local k-v store, local node and the local cloud with itself
 * as the only member.

src/main/java/water/Job.java

@@ -8,12 +8,15 @@
 public class Job extends Iced {
   // Global LIST of Jobs key.
   static final Key LIST = Key.make(Constants.BUILT_IN_KEY_JOBS, (byte) 0, Key.BUILT_IN_KEY);
+  private static final int KEEP_LAST_COUNT = 100;
+  public static final long CANCELLED_END_TIME = -1;
 
   // Per-job fields
   public final Key    _self; // Boolean read-only value; exists==>running, not-exists==>canceled/removed
   public final Key    _dest; // Key holding final value after job is removed
   public final String _description;
   public final long   _startTime;
+  public long         _endTime;
 
   public Key self() { return _self; }
   public Key dest() { return _dest; }
@@ -63,7 +66,7 @@ public float progress() {
     return dest != null ? dest.progress() : 0;
   }
 
-  // Block until the Job finishes.  
+  // Block until the Job finishes.
   // NOT F/J FRIENDLY, EATS THE THREAD until job completes.  Only use for web threads.
   public <T> T get() {
     // TODO through notifications?
@@ -78,9 +81,22 @@ public <T> T get() {
   }
 
   public void cancel() { cancel(_self); }
-  public static void cancel(Key self) {
+  public static void cancel(final Key self) {
     DKV.remove(self);
     DKV.write_barrier();
+    new TAtomic<List>() {
+      @Override public List atomic(List old) {
+        if( old == null ) old = new List();
+        Job[] jobs = old._jobs;
+        for( int i = 0; i < jobs.length; i++ ) {
+          if( jobs[i]._self.equals(self) ) {
+            jobs[i]._endTime = CANCELLED_END_TIME;
+            break;
+          }
+        }
+        return old;
+      }
+    }.fork(LIST);
   }
 
   public boolean cancelled() { return cancelled(_self); }
@@ -94,13 +110,29 @@ public void remove() {
       @Override public List atomic(List old) {
         if( old == null ) return null;
         Job[] jobs = old._jobs;
-        int i;
-        for( i = 0; i < jobs.length; i++ )
-          if( jobs[i]._self.equals(_self) )
+        for( int i = 0; i < jobs.length; i++ ) {
+          if( jobs[i]._self.equals(_self) ) {
+            if(jobs[i]._endTime != CANCELLED_END_TIME)
+              jobs[i]._endTime = System.currentTimeMillis();
+            break;
+          }
+        }
+        int count = jobs.length;
+        while(count > KEEP_LAST_COUNT) {
+          long min = Long.MAX_VALUE;
+          int n = -1;
+          for( int i = 0; i < jobs.length; i++ ) {
+            if(jobs[i]._endTime != 0 && jobs[i]._startTime < min) {
+              min = jobs[i]._startTime;
+              n = i;
+            }
+          }
+          if(n < 0)
             break;
-        if( i == jobs.length ) return null;
-        jobs[i] = jobs[jobs.length-1]; // Compact out the key from the list
-        old._jobs = Arrays.copyOf(jobs,jobs.length-1);
+          jobs[n] = jobs[--count];
+        }
+        if(count < jobs.length)
+          old._jobs = Arrays.copyOf(jobs, count);
         return old;
       }
     }.fork(LIST);

src/main/java/water/Log.java

@@ -15,12 +15,14 @@ protected SimpleDateFormat initialValue() {
       return format;
     }
   };
+  public static final String HOST;
   private static final String HOST_AND_PID;
   static boolean _dontDie;
   // @formatter:on
 
   static {
-    HOST_AND_PID = "" + padRight(H2O.findInetAddressForSelf().getHostAddress() + ", ", 17) + padRight(getPid() + ", ", 8);
+    HOST = H2O.findInetAddressForSelf().getHostAddress();
+    HOST_AND_PID = "" + padRight(HOST + ", ", 17) + padRight(getPid() + ", ", 8);
   }
 
   private static long getPid() {
@@ -57,11 +59,11 @@ public static void write(String s, Throwable t) {
   // Print to the original STDERR & die
   public static void die(String s) {
     System.err.println(s);
-    if(!_dontDie)
+    if( !_dontDie )
       System.exit(-1);
   }
 
-  static String padRight(String stringToPad, int size) {
+  public static String padRight(String stringToPad, int size) {
     StringBuilder strb = new StringBuilder(stringToPad);
 
     while( strb.length() < size )
@@ -71,12 +73,14 @@ static String padRight(String stringToPad, int size) {
     return strb.toString();
   }
 
-  static void initHeaders() {
-    System.setOut(new Wrapper(System.out));
-    System.setErr(new Wrapper(System.err));
+  public static void initHeaders() {
+    if( !(System.out instanceof Wrapper) ) {
+      System.setOut(new Wrapper(System.out));
+      System.setErr(new Wrapper(System.err));
+    }
   }
 
-  public static void write(PrintStream stream, String s, boolean headers) {
+  public static void unwrap(PrintStream stream, String s) {
     if( stream instanceof Wrapper )
       ((Wrapper) System.out).printlnParent(s);
     else