loggers grab RSS, cpu counters to track to cache-misses from perf stat

bench/BMLogs/bigLogger.sh

@@ -18,10 +18,10 @@ swapPerfFile=${OUTDIR}/`date        +%Y-%m-%d`"-sisoPerf_"$mach".csv"
 
 #headers
 cpuheader='time(s)'
-head -n 33 /proc/stat | tail -n 32 | awk -F' ' 'OFS="," {print $1}' > tmpfile
+head -n 33 /proc/stat | awk -F' ' 'OFS="," {print $1}' > tmpfile
 cpuheader=$cpuheader,`./transpose.sh tmpfile`
 rm tmpfile
-memheader='time(s),MemTotal,MemFree,Cached,Writeback'
+memheader='time(s),MemTotal,MemFree,Cached,Writeback,RSS'
 topheader='time(s),PID,USER,RES,%CPU,%MEM,COMMAND'
 netheader='time(s),dev,bytes,packets,errs,drop'
 sisoheader='time(s),si,so'
@@ -48,14 +48,15 @@ function echoLine {
         then
             line=`cat $1`
             echo $(( `date +%s` - $2 )),$6,$line >> $3
+        else
+            line=`cat $1`
+            echo $(( `date +%s` - $2 )),$line >> $3
         fi
-        line=`cat $1`
-        echo $(( `date +%s` - $2 )),$line >> $3
     else
         if [ $5 ]
         then
             line=`./transpose.sh $1 | awk -F, 'OFS="," {print $1,$2,$4,$17}'`
-            echo $(( `date +%s` - $2 )),$line >> $3
+            echo $(( `date +%s` - $2 )),$line,$6 >> $3
         else
             line=`./transpose.sh $1`
             echo $(( `date +%s` - $2 )),$line >> $3
@@ -86,6 +87,12 @@ done
 
 start=`cat starttime`
 while :; do
+    h2oPID=`ps -efww | grep h2o | grep 0xdiag| grep jar|awk '{print $2}' | xargs`
+    if [ -z $h2oPID ]
+    then
+        continue
+    fi
+
     #dump raw logs first
     ts=`date +"%Y-%m-%d-%H-%M-%S"`
     cat /proc/stat    >> ${rawLogs}/procstat/${ts}_procstat_${mach}
@@ -94,13 +101,13 @@ while :; do
     vmstat            >> ${rawLogs}/vmstat/${ts}_vmstat_${mach}
     top -b -n 1       >> ${rawLogs}/top/${ts}_top_${mach}
     a=1
-    for i in {0..34}
+    for i in {0..35}
     do
       TOTALS[$i]=0
     done
     while read -a CPU
     do a=$(($a+1));
-      if [ $a -eq 34 ]
+      if [ $a -eq 35 ]
       then
           break
       fi
@@ -128,7 +135,7 @@ while :; do
     echo $(( `date +%s` - $start )),$linecpu    >> $cpuPerfFile
     echo $(( `date +%s` - $start )),$lineidle   >> $idlePerfFile
     echo $(( `date +%s` - $start )),$lineiowait >> $iowaitPerfFile
-
+    RSS=`ps v  $h2oPID | awk -F' ' 'OFS="," {print $8}' | tail -n 1`
     cat /proc/meminfo      | awk -F' ' 'OFS="," {gsub(":","", $1); print $2}' > bmemTMP
     echo $pwd
     devstat=
@@ -140,13 +147,13 @@ while :; do
     esac
     grep $devstat /proc/net/dev  | awk -F' ' 'OFS="," {print $2,$3,$4,$5}'          > brecTMP
     grep $devstat /proc/net/dev  | awk -F' ' 'OFS="," {print $10,$11,$12,$13}'      > btraTMP
-    echoLine bmemTMP $start $memPerfFile         1 1
+    echoLine bmemTMP $start $memPerfFile         1 1 $RSS
     echoLine brecTMP $start $netReceivePerfFile  0 1 $devstat
     echoLine btraTMP $start $netTransmitPerfFile 0 1 $devstat
     #get top 10 processes from top and then just store them, may/not be interesting...
     ti="$(( `date +%s` - ${start} ))"
     top -b | head -n 17 | tail -n 10 | awk -v t=$ti -F' ' 'OFS="," {print t,$1,$2,$6,$9,$10,$12}' >> $topPerfFile
     vmstat | tail -n 1               | awk -v t=$ti -F' ' 'OFS="," {print t,$7,$8}'               >> $swapPerfFile
+    perf stat -x, -e instructions,cycles,cache-references,cache-misses,faults -a -o bTMP -p $h2oPID sleep 10
     rm b*TMP
-    sleep 10
 done

bench/BMLogs/littleLogger.sh

@@ -28,15 +28,15 @@ cachePerfFile=${OUTDIR}/$1/$1-`date       +%Y-%m-%d`"-cachePerf_"$mach".csv"
 
 #headers
 cpuheader='time(s)'
-head -n 33 /proc/stat | tail -n 32 | awk -F' ' 'OFS="," {print $1}' > tmpfile
+head -n 33 /proc/stat | awk -F' ' 'OFS="," {print $1}' > tmpfile
 cpuheader=$cpuheader,`./transpose.sh tmpfile`
 
 if [ -a tmpfile ]
 then
     rm tmpfile
 fi
 
-memheader='time(s),MemTotal,MemFree,Cached,Writeback'
+memheader='time(s),MemTotal,MemFree,Cached,Writeback,RSS'
 topheader='time(s),PID,USER,RES,%CPU,%MEM,COMMAND'
 netheader='time(s),dev,bytes,packets,errs,drop'
 sisoheader='time(s),si,so'
@@ -57,14 +57,15 @@ function echoLine {
         then
             line=`cat $1`
             echo $(( `date +%s` - $2 )),$6,$line >> $3
+        else
+            line=`cat $1`
+            echo $(( `date +%s` - $2 )),$line >> $3
         fi
-        line=`cat $1`
-        echo $(( `date +%s` - $2 )),$line >> $3
     else
         if [ $5 -eq 1 ]
         then
             line=`./transpose.sh $1 | awk -F, 'OFS="," {print $1,$2,$4,$17}'`
-            echo $(( `date +%s` - $2 )),$line >> $3
+            echo $(( `date +%s` - $2 )),$line,$6 >> $3
         else
             line=`./transpose.sh $1`
             echo $(( `date +%s` - $2 )),$line >> $3
@@ -82,7 +83,7 @@ checkExists $netTransmitPerfFile $netheader
 checkExists $swapPerfFile        $sisoheader
 checkExists $cachePerfFile       $cacheheader
 
-for i in {0..34}
+for i in {0..35}
 do
     PREVTOTALS[$i]=0
 done
@@ -96,13 +97,13 @@ while :; do
         continue
     fi
     a=1
-    for i in {0..34}
+    for i in {0..35}
     do
       TOTALS[$i]=0
     done
     while read -a CPU
     do a=$(($a+1));
-      if [ $a -eq 34 ]
+      if [ $a -eq 35 ]
       then
           break
       fi
@@ -143,7 +144,7 @@ while :; do
     grep $devstat /proc/net/dev  | awk -F' ' 'OFS="," {print $2,$3,$4,$5}'          > lrecTMP
     grep $devstat /proc/net/dev  | awk -F' ' 'OFS="," {print $10,$11,$12,$13}'      > ltraTMP
 
-    echoLine lmemTMP $start $memPerfFile         1 1
+    echoLine lmemTMP $start $memPerfFile         1 1 $RSS
     echoLine lrecTMP $start $netReceivePerfFile  0 1 $devstat
     echoLine ltraTMP $start $netTransmitPerfFile 0 1 $devstat
     #get top 10 processes from top and then just store them, may/not be interesting...

bench/BMscripts/gbmBench.py

@@ -1,5 +1,5 @@
 #GBM bench
-import os, sys, time, csv
+import os, sys, time, csv, string
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
@@ -12,6 +12,7 @@
              }
 build = ""
 debug = False
+json  = ""
 def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, depth, minrows, nbins, learnRate, response, row):
     debug = False
     bench = "bench"
@@ -101,6 +102,8 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
         gbm       = h2o_cmd.runGBM(parseResult = parseResult, noPoll=True, timeoutSecs=4800, **kwargs)
         h2o_jobs.pollWaitJobs(timeoutSecs=16000, pollTimeoutSecs=120, retryDelaySecs=5)
         gbmTime   = time.time() - gbmStart
+        cmd = 'cd ..; bash startloggers.sh ' + json + ' stop_'
+        os.system(cmd)
         row.update( {'gbmBuildTime'       : gbmTime,
                     })
         gbmTrainView = h2o_cmd.runGBMView(model_key='GBM('+f+')')
@@ -118,6 +121,7 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
     dat   = sys.argv.pop(-1)
     debug = sys.argv.pop(-1)
     build = sys.argv.pop(-1)
+    json  = sys.argv[-1].split('/')[-1]
     h2o.parse_our_args()
     h2o_hosts.build_cloud_with_hosts(enable_benchmark_log=False)
     fp    = 'Airlines' if 'Air' in dat else 'AllBedrooms'

bench/BMscripts/glm2Bench.py

@@ -1,5 +1,5 @@
 #GLM2 bench
-import os, sys, time, csv, re, requests
+import os, sys, time, csv, re, requests, string
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
@@ -12,6 +12,7 @@
              }
 build = ""
 debug = False
+json  = ""
 def doGLM2(f, folderPath, family, lambda_, alpha, nfolds, y, x, testFilehex, row, case_mode, case_val):
     debug = False
     bench = "bench"
@@ -96,6 +97,8 @@ def doGLM2(f, folderPath, family, lambda_, alpha, nfolds, y, x, testFilehex, row
         glm       = h2o_cmd.runGLM(parseResult = parseResult, timeoutSecs=1800, noPoll=True, **kwargs)
         h2o_jobs.pollWaitJobs(timeoutSecs=7200, pollTimeoutSecs=7200, retryDelaySecs=5)
         glmTime   = time.time() - glmStart
+        cmd = 'cd ..; bash startloggers.sh ' + json + ' stop_'
+        os.system(cmd)
         #glm       = h2o.nodes[0].inspect("GLM("+f+")")
         row.update( {'glm2BuildTime'       : glmTime,
                      #'AverageErrorOver10Folds'    : glm['glm_model']['validations'][0]['err'],
@@ -141,6 +144,7 @@ def doGLM2(f, folderPath, family, lambda_, alpha, nfolds, y, x, testFilehex, row
     dat   = sys.argv.pop(-1)
     debug = sys.argv.pop(-1)
     build = sys.argv.pop(-1)
+    json  = sys.argv[-1].split('/')[-1]
     h2o.parse_our_args()
     h2o_hosts.build_cloud_with_hosts()
     fp    = 'Airlines' if 'Air' in dat else 'AllBedrooms'

bench/BMscripts/glmBench.py

@@ -1,5 +1,5 @@
 #GLM bench
-import os, sys, time, csv, socket
+import os, sys, time, csv, socket, string
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
@@ -11,6 +11,7 @@
              }
 build = ""
 debug = False
+json  = ""
 def doGLM(f, folderPath, family, link, lambda_, alpha, nfolds, y, x, testFilehex, row):
     debug = False
     bench = "bench"
@@ -101,6 +102,8 @@ def doGLM(f, folderPath, family, link, lambda_, alpha, nfolds, y, x, testFilehex
                                             model_key   = params['destination_key'],
                                             timeoutSecs = 1800)
         scoreTime     = time.time() - glmScoreStart
+        cmd = 'cd ..; bash startloggers.sh ' + json + ' stop_'
+        os.system(cmd)
         if family == "binomial":
             row.update( {'scoreTime'          : scoreTime,
                          'AUC'                : glmScore['validation']['auc'],
@@ -121,6 +124,7 @@ def doGLM(f, folderPath, family, link, lambda_, alpha, nfolds, y, x, testFilehex
     dat   = sys.argv.pop(-1)
     debug = sys.argv.pop(-1)
     build = sys.argv.pop(-1)
+    json  = sys.argv[-1].split('/')[-1]
     h2o.parse_our_args()
     h2o_hosts.build_cloud_with_hosts()
     fp    = 'Airlines' if 'Air' in dat else 'AllBedrooms'

bench/BMscripts/kmeansBench.py

@@ -1,5 +1,5 @@
 #KMeans bench
-import os, sys, time, csv
+import os, sys, time, csv, string
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
@@ -11,6 +11,7 @@
              }   
 build = ""
 debug = False
+json  = ""
 def doKMeans(f, folderPath): 
     debug = False
     bench = "bench"
@@ -90,6 +91,8 @@ def doKMeans(f, folderPath):
                                             timeoutSecs=7200,
                                              **kwargs)
         kmeansTime      = time.time() - kmeansStart
+        cmd = 'cd ..; bash startloggers.sh ' + json + ' stop_'
+        os.system(cmd)
         row.update({'kmeansBuildTime' : kmeansTime})
         csvWrt.writerow(row)
     finally:
@@ -99,6 +102,7 @@ def doKMeans(f, folderPath):
     dat   = sys.argv.pop(-1)
     debug = sys.argv.pop(-1)
     build = sys.argv.pop(-1)
+    json  = sys.argv[-1].split('/')[-1]
     h2o.parse_our_args()
     h2o_hosts.build_cloud_with_hosts()
     fp    = 'Airlines' if 'Air' in dat else 'AllBedrooms'

bench/BMscripts/pcaBench.py

@@ -1,5 +1,5 @@
 #PCA bench
-import os, sys, time, csv
+import os, sys, time, csv, string
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_util
@@ -12,6 +12,7 @@
              }
 build = ""
 debug = False
+json = ""
 def doPCA(f, folderPath):
     debug = False
     bench = "bench"
@@ -84,7 +85,9 @@ def doPCA(f, folderPath):
                                    timeoutSecs = 7200, 
                                    **kwargs)
         pcaTime   = time.time() - pcaStart
-
+        cmd = 'cd ..; bash startloggers.sh ' + json + ' stop_'
+        #stop all loggers
+        os.system(cmd)
         row.update({'pcaBuildTime' : pcaTime})
         csvWrt.writerow(row)
     finally:
@@ -94,6 +97,7 @@ def doPCA(f, folderPath):
     dat   = sys.argv.pop(-1)
     debug = sys.argv.pop(-1)
     build = sys.argv.pop(-1)
+    json  = sys.argv[-1].split('/')[-1]
     fp    = 'Airlines' if 'Air' in dat else 'AllBedrooms'
     h2o.parse_our_args()
     h2o_hosts.build_cloud_with_hosts()

bench/BMscripts/summaryBench.py

@@ -1,5 +1,5 @@
 #summary bench
-import os, sys, time, csv
+import os, sys, time, csv, string
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_util
@@ -12,6 +12,7 @@
              }
 build = ""
 debug = False
+json  = ""
 def doSUM(f, folderPath):
     debug = False
     bench = "bench"
@@ -78,7 +79,8 @@ def doSUM(f, folderPath):
                                        timeoutSecs = 7200)
 
         sumTime             = time.time() - sumStart
-
+        cmd = 'cd ..; bash startloggers.sh ' + json + ' stop_'
+        os.system(cmd)
         row.update({'summaryBuildTime' : sumTime})
         csvWrt.writerow(row)
     finally:
@@ -88,6 +90,7 @@ def doSUM(f, folderPath):
     dat   = sys.argv.pop(-1)
     debug = sys.argv.pop(-1)
     build = sys.argv.pop(-1)
+    json  = sys.argv[-1].split('/')[-1]
     fp    = 'Airlines' if 'Air' in dat else 'AllBedrooms'
     h2o.parse_our_args()
     h2o_hosts.build_cloud_with_hosts()

bench/runBench.sh

@@ -36,26 +36,26 @@ function doAlgo {
     then
         python ${pyScript} -cj BMscripts/${JSON} ${h2oBuild} False Air1x;    wait; makeDead 2> /dev/null;
         zip -r ${archive}/${h2oBuild}-${DATE}-$1-Air1x sandbox/;             wait; rm -rf sandbox/;
-        bash startloggers.sh ${JSON} clear_; wait;
+        bash startloggers.sh ${JSON} clear_; wait; bash startloggers.sh ${JSON} changePhase $1; 
         python ${pyScript} -cj BMscripts/${JSON} ${h2oBuild} False Air10x;   wait; makeDead 2> /dev/null;
         zip -r ${archive}/${h2oBuild}-${DATE}-$1-Air10x sandbox/;            wait; rm -rf sandbox/;
-        if [ $2 = "gbm" ]
+        if [ $1 = "gbm" ]
         then
             continue
         fi
-        bash startloggers.sh ${JSON} clear_; wait;
+        bash startloggers.sh ${JSON} clear_; wait; bash startloggers.sh ${JSON} changePhase $1; 
         python ${pyScript} -cj BMscripts/${JSON} ${h2oBuild} False AllB1x;   wait; makeDead 2> /dev/null;
         zip -r ${archive}/${h2oBuild}-${DATE}-$1-AllB1x sandbox/;            wait; rm -rf sandbox/;
-        bash startloggers.sh ${JSON} clear_; wait;
+        bash startloggers.sh ${JSON} clear_; wait; bash startloggers.sh ${JSON} changePhase $1; 
         python ${pyScript} -cj BMscripts/${JSON} ${h2oBuild} False AllB10x;  wait; makeDead 2> /dev/null;
         zip -r ${archive}/${h2oBuild}-${DATE}-$1-AllB10x sandbox/;           wait; rm -rf sandbox/;
-        bash startloggers.sh ${JSON} clear_; wait;
+        bash startloggers.sh ${JSON} clear_; wait; bash startloggers.sh ${JSON} changePhase $1; 
         python ${pyScript} -cj BMscripts/${JSON} ${h2oBuild} False AllB100x; wait; makeDead 2> /dev/null;
         zip -r ${archive}/${h2oBuild}-${DATE}-$1-AllB100x sandbox/;          wait; rm -rf sandbox/;
-        bash startloggers.sh ${JSON} clear_; wait;
+        bash startloggers.sh ${JSON} clear_; wait; bash startloggers.sh ${JSON} changePhase $1; 
         python ${pyScript} -cj BMscripts/${JSON} ${h2oBuild} False Air100x;  wait; makeDead 2> /dev/null;
         zip -r ${archive}/${h2oBuild}-${DATE}-$1-Air100x sandbox/;           wait; rm -rf sandbox/;
-        bash startloggers.sh ${JSON} clear_; wait;
+        bash startloggers.sh ${JSON} clear_; wait; bash startloggers.sh ${JSON} changePhase $1; 
     else
         JSON2=161
         echo "Doing KMeans.. Using ${JSON2} config file..."