update archiving, turn off extra logging

xiangchenm · Oct 17, 2013 · 5efade6 · 5efade6
1 parent 9d6da3d
commit 5efade6
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 9 deletions.
diff --git a/bench/BMscripts/pcaBench.py b/bench/BMscripts/pcaBench.py
@@ -12,10 +12,12 @@
 build = ""
 
 def doPCA(fs, folderPath):
-    benchmarkLogging = ['cpu','disk', 'network', 'iostats']
+    #benchmarkLogging = ['cpu','disk', 'network', 'iostats']
+    benchmarkLogging = None
     date = '-'.join([str(x) for x in list(time.localtime())][0:3])
     for f in fs['train']:
         #if f != 'AirlinesTrain1x': continue
+        retryDelaySecs = 5 if f == 'AirlinesTrain1x' else 30
         h2o.cloudPerfH2O.switch_logfile(location='./BMLogs/'+build+ '/' + date, log='PCA'+f+'.csv')
         print "Doing PCA on ", f
         overallWallStart = time.time()
@@ -40,7 +42,7 @@ def doPCA(fs, folderPath):
             headerKey = h2i.find_key(hK)
             h2o.cloudPerfH2O.message("=========PARSE TRAIN========")
             parseResult = h2i.import_parse(bucket='home-0xdiag-datasets', path=csvPathname, schema='local', hex_key=hex_key,header=1, header_from_file=headerKey, separator=44,
-                timeoutSecs=7200,retryDelaySecs=5,pollTimeoutSecs=7200, benchmarkLogging=benchmarkLogging)
+                timeoutSecs=7200,retryDelaySecs=retryDelaySecs,pollTimeoutSecs=7200, benchmarkLogging=benchmarkLogging)
             h2o.cloudPerfH2O.message("=========END PARSE TRAIN========")
             parseWallTime = time.time() - trainParseWallStart
             print "Parsing training file took ", parseWallTime ," seconds." 

diff --git a/bench/runBench.sh b/bench/runBench.sh
@@ -35,7 +35,7 @@ function PCA() {
     pyScript="BMscripts/pcaBench.py"
     python ${pyScript} --config_json BMscripts/${JSON} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-PCA sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-PCA sandbox/
     wait
     rm -rf sandbox/
 }
@@ -44,7 +44,7 @@ function KMeans() {
     pyScript="BMscripts/kmeansBench.py"
     python ${pyScript} --config_json BMscripts/${JSON} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-KMEANS sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-KMEANS sandbox/
     wait
     rm -rf sandbox/
 }
@@ -53,7 +53,7 @@ function BigKMeans() {
     pyScript="BMscripts/bigkmeansBench.py"
     python ${pyScript} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-BIGKMEANS sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-BIGKMEANS sandbox/
     wait
     rm sandbox/
 }
@@ -62,7 +62,7 @@ function GLM() {
     pyScript="BMscripts/glmBench.py"
     python ${pyScript} --config_json BMscripts/${JSON} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-GLM sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-GLM sandbox/
     wait
     rm -rf sandbox/
 }
@@ -71,7 +71,7 @@ function GLM2() {
     pyScript="BMscripts/glm2Bench.py"
     python ${pyScript} --config_json BMscripts/${JSON} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-GLM2 sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-GLM2 sandbox/
     wait
     rm -rf sandbox/
 }
@@ -81,7 +81,7 @@ function GBM() {
     pyScript="BMscripts/gbmBench.py"
     python ${pyScript} --config_json BMscripts/${JSON} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-GBM sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-GBM sandbox/
     wait
     rm -rf sandbox/
 }
@@ -90,7 +90,7 @@ function GBMGrid() {
     pyScript="BMscripts/gbmgridBench.py"
     python ${pyScript} --config_json BMscripts/${JSON} ${h2oBuild}
     wait
-    zip ${archive}/${h2oBuild}-${DATE}-GBMGrid sandbox/
+    zip -r  ${archive}/${h2oBuild}-${DATE}-GBMGrid sandbox/
     wait
     rm -rf sandbox/
 }