Skip to content

Commit c08c3fe

Browse files
spennihanaspennihana
committed
merge benchmark changes to master
1 parent 7a95bca commit c08c3fe

14 files changed

+373
-87
lines changed

‎bench/BMLogs/bigLogger.sh

+18-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22

33
OUTDIR='BigLoggerFiles'
4-
rawLogs=${OUTDIR}/rawLogs
4+
rawLogs=rawLogs
55

66
#last 3 digits of inet addr
77
mach=`ifconfig | grep -o "inet addr:192.168.1.[0-9]*" | grep -o 192.168.1.* | awk -F'.' '{print $4}'`
@@ -23,7 +23,7 @@ cpuheader=$cpuheader,`./transpose.sh tmpfile`
2323
rm tmpfile
2424
memheader='time(s),MemTotal,MemFree,Cached,Writeback'
2525
topheader='time(s),PID,USER,RES,%CPU,%MEM,COMMAND'
26-
netheader='time(s),bytes,packets,errs,drop'
26+
netheader='time(s),dev,bytes,packets,errs,drop'
2727
sisoheader='time(s),si,so'
2828

2929
function checkExists {
@@ -44,6 +44,11 @@ function checkDExists {
4444
function echoLine {
4545
if [ $4 -eq 0 ]
4646
then
47+
if [ $5 -eq 1 ]
48+
then
49+
line=`cat $1`
50+
echo $(( `date +%s` - $2 )),$6,$line >> $3
51+
fi
4752
line=`cat $1`
4853
echo $(( `date +%s` - $2 )),$line >> $3
4954
else
@@ -126,11 +131,18 @@ while :; do
126131

127132
cat /proc/meminfo | awk -F' ' 'OFS="," {gsub(":","", $1); print $2}' > bmemTMP
128133
echo $pwd
129-
grep lo /proc/net/dev | awk -F' ' 'OFS="," {print $2,$3,$4,$5}' > brecTMP
130-
grep lo /proc/net/dev | awk -F' ' 'OFS="," {print $10,$11,$12,$13}' > btraTMP
134+
devstat=
135+
case "$mach" in
136+
161) devstat="eth1" ;;
137+
162) devstat="eth2" ;;
138+
163) devstat="eth3" ;;
139+
164) devstat="eth3" ;;
140+
esac
141+
grep $devstat /proc/net/dev | awk -F' ' 'OFS="," {print $2,$3,$4,$5}' > brecTMP
142+
grep $devstat /proc/net/dev | awk -F' ' 'OFS="," {print $10,$11,$12,$13}' > btraTMP
131143
echoLine bmemTMP $start $memPerfFile 1 1
132-
echoLine brecTMP $start $netReceivePerfFile 0
133-
echoLine btraTMP $start $netTransmitPerfFile 0
144+
echoLine brecTMP $start $netReceivePerfFile 0 1 $devstat
145+
echoLine btraTMP $start $netTransmitPerfFile 0 1 $devstat
134146
#get top 10 processes from top and then just store them, may/not be interesting...
135147
ti="$(( `date +%s` - ${start} ))"
136148
top -b | head -n 17 | tail -n 10 | awk -v t=$ti -F' ' 'OFS="," {print t,$1,$2,$6,$9,$10,$12}' >> $topPerfFile

‎bench/BMLogs/littleLogger.sh

+50-16
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,24 @@ then
77
mkdir ${OUTDIR}
88
fi
99

10+
if [ ! -d ${OUTDIR}/$1 ]
11+
then
12+
mkdir ${OUTDIR}/$1
13+
fi
14+
1015
#last 3 digits of inet addr
1116
mach=`ifconfig | grep -o "inet addr:192.168.1.[0-9]*" | grep -o 192.168.1.* | awk -F'.' '{print $4}'`
1217

1318
#log files
14-
cpuPerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-cpuPerf_"$mach".csv"
15-
idlePerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-idlePerf_"$mach".csv"
16-
iowaitPerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-iowaitPerf_"$mach".csv"
17-
memPerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-memPerf_"$mach".csv"
18-
topPerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-topPerf_"$mach".csv"
19-
netReceivePerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-netRPerf_"$mach".csv"
20-
netTransmitPerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-netTPerf_"$mach".csv"
21-
swapPerfFile=${OUTDIR}/$1-`date +%Y-%m-%d`"-sisoPerf_"$mach".csv"
19+
cpuPerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-cpuPerf_"$mach".csv"
20+
idlePerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-idlePerf_"$mach".csv"
21+
iowaitPerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-iowaitPerf_"$mach".csv"
22+
memPerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-memPerf_"$mach".csv"
23+
topPerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-topPerf_"$mach".csv"
24+
netReceivePerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-netRPerf_"$mach".csv"
25+
netTransmitPerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-netTPerf_"$mach".csv"
26+
swapPerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-sisoPerf_"$mach".csv"
27+
cachePerfFile=${OUTDIR}/$1/$1-`date +%Y-%m-%d`"-cachePerf_"$mach".csv"
2228

2329
#headers
2430
cpuheader='time(s)'
@@ -32,8 +38,9 @@ fi
3238

3339
memheader='time(s),MemTotal,MemFree,Cached,Writeback'
3440
topheader='time(s),PID,USER,RES,%CPU,%MEM,COMMAND'
35-
netheader='time(s),bytes,packets,errs,drop'
41+
netheader='time(s),dev,bytes,packets,errs,drop'
3642
sisoheader='time(s),si,so'
43+
cacheheader='time(s),instructions,cycles,cache-references,cache-misses,faults'
3744

3845
function checkExists {
3946
if [ ! -a $1 ]
@@ -46,10 +53,15 @@ function checkExists {
4653
function echoLine {
4754
if [ $4 -eq 0 ]
4855
then
56+
if [ $5 -eq 1 ]
57+
then
58+
line=`cat $1`
59+
echo $(( `date +%s` - $2 )),$6,$line >> $3
60+
fi
4961
line=`cat $1`
5062
echo $(( `date +%s` - $2 )),$line >> $3
5163
else
52-
if [ $5 ]
64+
if [ $5 -eq 1 ]
5365
then
5466
line=`./transpose.sh $1 | awk -F, 'OFS="," {print $1,$2,$4,$17}'`
5567
echo $(( `date +%s` - $2 )),$line >> $3
@@ -68,6 +80,7 @@ checkExists $topPerfFile $topheader
6880
checkExists $netReceivePerfFile $netheader
6981
checkExists $netTransmitPerfFile $netheader
7082
checkExists $swapPerfFile $sisoheader
83+
checkExists $cachePerfFile $cacheheader
7184

7285
for i in {0..34}
7386
do
@@ -76,6 +89,12 @@ done
7689

7790
start=`cat starttime`
7891
while :; do
92+
h2oPID=`ps -efww | grep h2o | grep 0xdiag| grep jar|awk '{print $2}' | xargs`
93+
94+
if [ -z $h2oPID ]
95+
then
96+
continue
97+
fi
7998
a=1
8099
for i in {0..34}
81100
do
@@ -111,19 +130,34 @@ while :; do
111130
echo $(( `date +%s` - $start )),$linecpu >> $cpuPerfFile
112131
echo $(( `date +%s` - $start )),$lineidle >> $idlePerfFile
113132
echo $(( `date +%s` - $start )),$lineiowait >> $iowaitPerfFile
114-
133+
#RSS -> ps v 25707 | awk -F' ' 'OFS="," {print $8}'
134+
RSS=`ps v $h2oPID | awk -F' ' 'OFS="," {print $8}' | tail -n 1`
115135
cat /proc/meminfo | awk -F' ' 'OFS="," {gsub(":","", $1); print $2}' > lmemTMP
116-
grep lo /proc/net/dev | awk -F' ' 'OFS="," {print $2,$3,$4,$5}' > lrecTMP
117-
grep lo /proc/net/dev | awk -F' ' 'OFS="," {print $10,$11,$12,$13}' > ltraTMP
136+
devstat=
137+
case "$mach" in
138+
161) devstat="eth1" ;;
139+
162) devstat="eth2" ;;
140+
163) devstat="eth3" ;;
141+
164) devstat="eth3" ;;
142+
esac
143+
grep $devstat /proc/net/dev | awk -F' ' 'OFS="," {print $2,$3,$4,$5}' > lrecTMP
144+
grep $devstat /proc/net/dev | awk -F' ' 'OFS="," {print $10,$11,$12,$13}' > ltraTMP
145+
118146
echoLine lmemTMP $start $memPerfFile 1 1
119-
echoLine lrecTMP $start $netReceivePerfFile 0
120-
echoLine ltraTMP $start $netTransmitPerfFile 0
147+
echoLine lrecTMP $start $netReceivePerfFile 0 1 $devstat
148+
echoLine ltraTMP $start $netTransmitPerfFile 0 1 $devstat
121149
#get top 10 processes from top and then just store them, may/not be interesting...
122150
ti="$(( `date +%s` - ${start} ))"
123151
top -b | head -n 17 | tail -n 10 | awk -v t=$ti -F' ' 'OFS="," {print t,$1,$2,$6,$9,$10,$12}' >> $topPerfFile
124152
vmstat | tail -n 1 | awk -v t=$ti -F' ' 'OFS="," {print t,$7,$8}' >> $swapPerfFile
153+
perf stat -x, -e instructions,cycles,cache-references,cache-misses,faults -a -o lTMP -p $h2oPID sleep 10
154+
155+
tail -n 5 lTMP > lcacheTMP
156+
./transpose.sh lcacheTMP | head -n 1 >> debug_out
157+
line=`./transpose.sh lcacheTMP | head -n 1`
158+
echo $(( `date +%s` - ${start})),$line >> $cachePerfFile
159+
125160
rm l*TMP
126-
sleep 10
127161
done
128162

129163

‎bench/BMscripts/161

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"use_home_for_ice": true,
99

1010
"h2o_per_host": 1,
11-
"java_heap_GB": 100,
11+
"java_heap_GB": 240,
1212

1313
"hdfs_name_node": "192.168.1.176",
1414
"hdfs_version": "cdh3",

‎bench/BMscripts/161_164

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"use_home_for_ice": true,
99

1010
"h2o_per_host": 1,
11-
"java_heap_GB": 150,
11+
"java_heap_GB": 200,
1212

1313
"hdfs_name_node": "192.168.1.176",
1414
"hdfs_version": "cdh3",

‎bench/BMscripts/bigkmeansBench.py

+12-7
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44
sys.path.extend(['.','..'])
55
import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
66

7-
csv_header = ('h2o_build','nMachines','nJVMs','Xmx/JVM','dataset','nRows','nCols','parseWallTime','kmeansBuildTime')
7+
csv_header = ('h2o_build','nMachines','nJVMs','Xmx/JVM','dataset','nRows','nCols','parseWallTime','k','max_iter','init','kmeansBuildTime')
88

99
build = ""
1010
debug = False #TODO(spencer): make a debug mode
1111
def doKMeans():
12-
f = ""
12+
f = "15sphers"
1313
date = '-'.join([str(x) for x in list(time.localtime())][0:3])
1414
overallWallStart = time.time()
15-
kmeansbenchcsv = 'benchmarks/'+build+'/'+date+'/bigkmeansbench.csv'
15+
kmeansbenchcsv = 'benchmarks/'+build+'/'+date+'/kmeansbench.csv'
1616
if not os.path.exists(kmeansbenchcsv):
1717
output = open(kmeansbenchcsv,'w')
1818
output.write(','.join(csv_header)+'\n')
@@ -23,12 +23,12 @@ def doKMeans():
2323
try:
2424
java_heap_GB = h2o.nodes[0].java_heap_GB
2525
#Train File Parsing#
26-
importFolderPath = "kmeans_big"
26+
importFolderPath = "0xdiag/datasets/kmeans_big"
2727
csvPathname = 'syn_sphere15_2711545732row_6col_180GB_from_7x.csv'
2828
hex_key = csvPathname + '.hex'
2929
trainParseWallStart = time.time()
30-
parseResult = h2i.import_parse(bucket = 'home3-0xdiag-datasets',
31-
path = csvPathname,
30+
parseResult = h2i.import_parse(bucket = '/home3',
31+
path = importFolderPath + '/' + csvPathname,
3232
schema = 'local',
3333
hex_key = hex_key,
3434
separator = 44,
@@ -51,6 +51,9 @@ def doKMeans():
5151
'nRows' : inspect['num_rows'],
5252
'nCols' : inspect['num_cols'],
5353
'parseWallTime' : parseWallTime,
54+
'k' : 15,
55+
'max_iter' : 100,
56+
'init' : 'Furthest',
5457
}
5558

5659
params = {'source_key' : hex_key,
@@ -72,8 +75,10 @@ def doKMeans():
7275
output.close()
7376

7477
if __name__ == '__main__':
78+
dat = sys.argv.pop(-1)
7579
debug = sys.argv.pop(-1)
7680
build = sys.argv.pop(-1)
7781
h2o.parse_our_args()
78-
h2o.build_cloud(1, java_heap_GB=240, enable_benchmark_log=False)
82+
h2o_hosts.build_cloud_with_hosts()
83+
doKMeans()
7984
h2o.tear_down_cloud()

‎bench/BMscripts/gbmBench.py

+16-11
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
sys.path.extend(['.','..'])
55
import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
66

7-
csv_header = ('h2o_build','nMachines','nJVMs','Xmx/JVM','dataset','nTrainRows','nTestRows','nCols','trainParseWallTime','classification','gbmBuildTime','Error')
7+
csv_header = ('h2o_build','nMachines','nJVMs','Xmx/JVM','dataset','nTrainRows','nTestRows','nCols','trainParseWallTime','nTrees','minRows','maxDepth','learnRate','classification','gbmBuildTime','Error')
88

99
files = {'Airlines' : {'train': ('AirlinesTrain1x', 'AirlinesTrain10x', 'AirlinesTrain100x'), 'test' : 'AirlinesTest'},
1010
'AllBedrooms': {'train': ('AllBedroomsTrain1x', 'AllBedroomsTrain10x', 'AllBedroomsTrain100x'), 'test' : 'AllBedroomsTest'},
@@ -13,7 +13,7 @@
1313
build = ""
1414
debug = False
1515
def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, depth, minrows, nbins, learnRate, response, row):
16-
#debug = False
16+
debug = False
1717
bench = "bench"
1818
if debug:
1919
print "Doing GBM DEBUG"
@@ -52,18 +52,18 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
5252
header = 1,
5353
header_from_file = headerKey,
5454
separator = 44,
55-
timeoutSecs = 7200,
55+
timeoutSecs = 16000,
5656
retryDelaySecs = 5,
57-
pollTimeoutSecs = 7200,
57+
pollTimeoutSecs = 16000,
5858
noPoll = True,
5959
doSummary = False
6060
)
61-
h2o_jobs.pollWaitJobs(timeoutSecs=7200, pollTimeoutSecs=7200, retryDelaySecs=5)
61+
h2o_jobs.pollWaitJobs(timeoutSecs=16000, pollTimeoutSecs=16000, retryDelaySecs=5)
6262
parseWallTime = time.time() - trainParseWallStart
6363
print "Parsing training file took ", parseWallTime ," seconds."
6464
h2o.beta_features = False #make sure false for the inspect as well!
65-
inspect_train = h2o.nodes[0].inspect(hex_key, timeoutSecs=7200)
66-
inspect_test = h2o.nodes[0].inspect(testFilehex, timeoutSecs=7200)
65+
inspect_train = h2o.nodes[0].inspect(hex_key, timeoutSecs=16000)
66+
inspect_test = h2o.nodes[0].inspect(testFilehex, timeoutSecs=16000)
6767
h2o.beta_features = True #ok, can be true again
6868
nMachines = 1 if len(h2o_hosts.hosts) is 0 else len(h2o_hosts.hosts)
6969
row.update( {'h2o_build' : build,
@@ -75,6 +75,10 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
7575
'nTestRows' : inspect_test['num_rows'],
7676
'nCols' : inspect_train['num_cols'],
7777
'trainParseWallTime' : parseWallTime,
78+
'nTrees' : ntrees,
79+
'minRows' : minrows,
80+
'maxDepth' : depth,
81+
'learnRate' : learnRate,
7882
'classification' : classification,
7983
})
8084

@@ -95,7 +99,7 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
9599
gbmStart = time.time()
96100
#TODO(spencer): Uses jobs to poll for gbm completion
97101
gbm = h2o_cmd.runGBM(parseResult = parseResult, noPoll=True, timeoutSecs=4800, **kwargs)
98-
h2o_jobs.pollWaitJobs(timeoutSecs=7200, pollTimeoutSecs=120, retryDelaySecs=5)
102+
h2o_jobs.pollWaitJobs(timeoutSecs=16000, pollTimeoutSecs=120, retryDelaySecs=5)
99103
gbmTime = time.time() - gbmStart
100104
row.update( {'gbmBuildTime' : gbmTime,
101105
})
@@ -119,6 +123,7 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
119123
fp = 'Airlines' if 'Air' in dat else 'AllBedrooms'
120124
bench = "bench"
121125
h2o.beta_features = True
126+
debug = False
122127
if debug:
123128
bench = "bench/debug"
124129

@@ -137,7 +142,7 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
137142
h2i.import_only(bucket='home-0xdiag-datasets', path=headerPathname)
138143
headerKey = h2i.find_key(hK)
139144
testFile = h2i.import_parse(bucket='home-0xdiag-datasets', path=bench+'/Airlines/AirlinesTest.csv', schema='local', hex_key="atest.hex", header=1, header_from_file=headerKey, separator=44, noPoll=True,doSummary=False)
140-
h2o_jobs.pollWaitJobs(timeoutSecs=7200, pollTimeoutSecs=7200, retryDelaySecs=5)
145+
h2o_jobs.pollWaitJobs(timeoutSecs=16000, pollTimeoutSecs=16000, retryDelaySecs=5)
141146
elapsedAirlinesTestParse = time.time() - airlinesTestParseStart
142147
row = {'testParseWallTime' : elapsedAirlinesTestParse}
143148
response = 'IsDepDelayed'
@@ -163,7 +168,7 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
163168
h2i.import_only(bucket='home-0xdiag-datasets', path=headerPathname)
164169
headerKey = h2i.find_key(hK)
165170
testFile = h2i.import_parse(bucket='home-0xdiag-datasets', path=bench+'/AllBedrooms/AllBedroomsTest.csv', schema='local', hex_key="allBTest.hex", header=1, header_from_file=headerKey, separator=44,noPoll=True,doSummary=False)
166-
h2o_jobs.pollWaitJobs(timeoutSecs=7200, pollTimeoutSecs=7200, retryDelaySecs=5)
171+
h2o_jobs.pollWaitJobs(timeoutSecs=16000, pollTimeoutSecs=16000, retryDelaySecs=5)
167172
elapsedAllBedroomsTestParse = time.time() - allBedroomsTestParseStart
168173
row = {'testParseWallTime' : elapsedAllBedroomsTestParse}
169174
response = 'medrent'
@@ -188,7 +193,7 @@ def doGBM(f, folderPath, ignored_cols, classification, testFilehex, ntrees, dept
188193
#h2i.import_only(bucket='home-0xdiag-datasets', path=headerPathname)
189194
#headerKey = h2i.find_key(hK)
190195
#testFile = h2i.import_parse(bucket='home-0xdiag-datasets', path=bench+'/CovType/CovTypeTest.csv', schema='local', hex_key="covTtest.hex", header=1, header_from_file=headerKey, separator=44, noPoll=True,doSummary=False)
191-
#h2o_jobs.pollWaitJobs(timeoutSecs=7200, pollTimeoutSecs=7200, retryDelaySecs=5)
196+
#h2o_jobs.pollWaitJobs(timeoutSecs=16000, pollTimeoutSecs=16000, retryDelaySecs=5)
192197
#elapsedCovTypeTestParse = time.time() - covTypeTestParseStart
193198
#row = {'testParseWallTime' : elapsedCovTypeTestParse}
194199
#response = 'C55'

0 commit comments

Comments
 (0)