Persistence spring cleaning

TATABOX42 · May 29, 2013 · 50450a6 · 50450a6
1 parent c697444
commit 50450a6
Show file tree

Hide file tree

Showing 32 changed files with 1,102 additions and 953 deletions.
diff --git a/.classpath b/.classpath
@@ -14,7 +14,6 @@
 	<classpathentry kind="lib" path="lib/poi/poi-3.8-20120326.jar" sourcepath="lib/poi/poi-3.8-sources.jar"/>
 	<classpathentry kind="lib" path="lib/poi/poi-ooxml-3.8-20120326.jar"/>
 	<classpathentry kind="lib" path="lib/poi/poi-ooxml-schemas-3.8-20120326.jar"/>
-	<classpathentry kind="lib" path="lib/trove/trove-3.0.3.jar"/>
 	<classpathentry kind="lib" path="lib/s3/aws-java-sdk-1.3.27.jar" sourcepath="lib/s3/aws-java-sdk-1.3.27-sources.jar"/>
 	<classpathentry kind="lib" path="lib/jama/Jama.jar"/>
 	<classpathentry kind="lib" path="lib/javassist.jar" sourcepath="lib/javassist-sources.jar"/>

diff --git a/py/runBeforePush.sh b/py/runBeforePush.sh
diff --git a/src/main/java/water/H2O.java b/src/main/java/water/H2O.java
@@ -7,13 +7,11 @@
 import java.util.*;
 
 import jsr166y.*;
-import water.r.Shell;
-import water.api.Constants.Schemes;
 import water.exec.Function;
-import water.hdfs.HdfsLoader;
 import water.nbhm.NonBlockingHashMap;
 import water.parser.ParseDataset;
-import water.store.s3.PersistS3;
+import water.persist.*;
+import water.r.Shell;
 import water.util.*;
 import water.util.Log.Tag.Sys;
 
@@ -28,7 +26,6 @@
 * @version 1.0
 */
 public final class H2O {
-
   static boolean _hdfsActive = false;
 
   public static final String VERSION = "0.3";
@@ -53,6 +50,9 @@ public final class H2O {
   public static H2ONode SELF = null;
   public static InetAddress SELF_ADDRESS;
 
+  public static final String DEFAULT_ICE_ROOT = "/tmp";
+  public static URI ICE_ROOT;
+
   // Initial arguments
   public static String[] ARGS;
 
@@ -516,12 +516,22 @@ public static void main( String[] args ) {
     ARGS = arguments.toStringArray();
     ParseDataset.PLIMIT = OPT_ARGS.pparse_limit;
 
+    // Get ice path before loading Log or Persist class
+    String ice = DEFAULT_ICE_ROOT;
+    if( OPT_ARGS.ice_root != null ) ice = OPT_ARGS.ice_root.replace("\\", "/");
+    try {
+      ICE_ROOT = new URI(ice);
+    } catch(URISyntaxException ex) {
+      throw new RuntimeException("Invalid ice_root: " + ice + ", " + ex.getMessage());
+    }
+
     SELF_ADDRESS = findInetAddressForSelf();
 
     //if (OPT_ARGS.rshell.equals("false"))
     Log.wrap(); // Logging does not wrap when the rshell is on.
 
-    startLocalNode(); // start the local node
+    // Start the local node
+    startLocalNode();
     // Load up from disk and initialize the persistence layer
     initializePersistence();
     // Start network services, including heartbeats & Paxos
@@ -837,14 +847,13 @@ public static List<FlatFileEntry> parseFlatFile( File f ) {
   }
 
   static void initializePersistence() {
-    PersistIce.initialize();
-    PersistNFS.initialize();
-    HdfsLoader.initialize();
+    HdfsLoader.loadJars();
     if( OPT_ARGS.aws_credentials != null ) {
       try {
         PersistS3.getClient();
       } catch( IllegalArgumentException e ) { Log.err(e); }
     }
+    Persist.initialize();
   }
 
 
@@ -885,12 +894,8 @@ static boolean lazyPersist(){ // free disk > our DRAM?
       return H2O.SELF._heartbeat.get_free_disk() > MemoryManager.MEM_MAX;
     }
     static boolean isDiskFull(){ // free disk space < 5K?
-      if(Schemes.HDFS.equals(PersistIce.ROOT.getScheme())) {
-        // TODO actual check? for now assume HDFS always happy
-        return false;
-      }
-      File f = new File(PersistIce.ROOT.getPath());
-      return f.getUsableSpace() < (5 << 10);
+      long space = Persist.getIce().getUsableSpace();
+      return space != Persist.UNKNOWN && space < (5 << 10);
     }
     public void run() {
       boolean diskFull = false;
@@ -989,8 +994,8 @@ public void run() {
               if( m == null ) m = val.rawMem();
               if( m != null ) cleaned += m.length;
             } catch(IOException e) {
-              if( isDiskFull() ) // disk full?
-                Log.warn(Sys.CLEAN,"Disk full! Disabling swapping to disk." + ((force)?" Memory low! Please free some space in " + PersistIce.ROOT+"!":""));
+              if( isDiskFull() )
+                Log.warn(Sys.CLEAN,"Disk full! Disabling swapping to disk." + (force?" Memory low! Please free some space in " + Persist.getIce().getPath() + "!":""));
               else
                 Log.warn(Sys.CLEAN,"Disk swapping failed! " + e.getMessage());
               // Something is wrong so mark disk as full anyways so we do not

diff --git a/src/main/java/water/HeartBeatThread.java b/src/main/java/water/HeartBeatThread.java
@@ -1,11 +1,10 @@
 package water;
 
-import java.io.File;
 import java.lang.management.ManagementFactory;
 
 import javax.management.*;
 
-import water.api.Constants.Schemes;
+import water.persist.Persist;
 import water.util.Log;
 
 /**
@@ -99,14 +98,8 @@ public void run() {
 
       // get the usable and total disk storage for the partition where the
       // persistent KV pairs are stored
-      if (PersistIce.ROOT==null || Schemes.HDFS.equals(PersistIce.ROOT.getScheme())) {
-        hb.set_free_disk(0); // not applicable
-        hb.set_max_disk(0); // not applicable
-      } else {
-        File f = new File(PersistIce.ROOT.getPath());
-        hb.set_free_disk(f.getUsableSpace());
-        hb.set_max_disk(f.getTotalSpace());
-      }
+      hb.set_free_disk(Persist.getIce().getUsableSpace());
+      hb.set_max_disk(Persist.getIce().getTotalSpace());
 
       // Announce what Cloud we think we are in.
       // Publish our health as well.