Skip to content

Commit

Permalink
I think this makes --usecloud (-uc) work
Browse files Browse the repository at this point in the history
  • Loading branch information
Kevin Normoyle committed Nov 7, 2014
1 parent b373337 commit 6e97168
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 12 deletions.
1 change: 0 additions & 1 deletion py/h2o.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,5 +118,4 @@ def init(*args, **kwargs):
nodes = build_cloud(*args, **kwargs)
else:
nodes = h2o_hosts.build_cloud_with_hosts(*args, **kwargs)
print "kevin:", len(nodes)

1 change: 0 additions & 1 deletion py/h2o_bc.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,6 @@ def build_cloud(node_count=1, base_port=None, hosts=None,
# potentially passed in kwargs
# hdfs_version='cdh4', hdfs_config=None, hdfs_name_node='172.16.1.176',

assert 1==0
nodeList = build_cloud_with_json(h2o_nodes_json=nodesJsonPathname)
return nodeList

Expand Down
21 changes: 13 additions & 8 deletions py/h2o_fc.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@

from h2o_test import get_sandbox_name, check_sandbox_for_errors, dump_json, verboseprint

#*********************************************************************************************
#*******************************************************************************
# duplicate do_json_request here, because the normal one is a method on a h2o node object which
# doesn't exist yet. copied this from find_cloud.py
def create_url(addr, port, loc):
return 'http://%s:%s/%s' % (addr, port, loc)

#*********************************************************************************************
#*******************************************************************************
def do_json_request(addr=None, port=None, jsonRequest=None, params=None, timeout=7, **kwargs):
if params is not None:
paramsStr = '?' + '&'.join(['%s=%s' % (k,v) for (k,v) in params.items()])
Expand All @@ -35,8 +35,8 @@ def do_json_request(addr=None, port=None, jsonRequest=None, params=None, timeou
rjson = None
emsg = "ERROR: json got ConnectionError or other exception"
# Rethrow the exception after we've checked for stack trace from h2o.
# Out of memory errors maybe don't show up right away? so we should wait for h2o
# to get it out to h2o stdout.
# Out of memory errors maybe don't show up right away?
# so we should wait for h2o to get it out to h2o stdout.
# Don't want to rely on cloud teardown to check because there's no delay,
# and we don't want to delay all cloud teardowns by waiting.
exc_info = sys.exc_info()
Expand All @@ -51,7 +51,10 @@ def do_json_request(addr=None, port=None, jsonRequest=None, params=None, timeou
return rjson

#*********************************************************************************************
def probe_node(line, h2oNodes, expectedSize):
def probe_node(line, h2oNodes, expectedSize, hdfsSetup):

(hdfs_version, hdfs_config, hdfs_name_node) = hdfsSetup

http_addr, sep, port = line.rstrip('\n').partition(":")
http_addr = http_addr.lstrip('/') # just in case it's an old-school flatfile format with leading /
if port == '':
Expand Down Expand Up @@ -89,8 +92,9 @@ def probe_node(line, h2oNodes, expectedSize):
print "tot_mem_bytes (GB):", "%0.2f" % ((n['tot_mem_bytes']+0.0)/(1024*1024*1024))
java_heap_GB = (n['tot_mem_bytes']+0.0)/(1024*1024*1024)
java_heap_GB = int(round(java_heap_GB,0))
num_cpus = n['num_cpus']
print "java_heap_GB:", java_heap_GB
print 'num_cpus:', n['num_cpus']
print 'num_cpus:', num_cpus

java_heap_GB_list.append(java_heap_GB)
num_cpus_list.append(num_cpus)
Expand Down Expand Up @@ -177,16 +181,17 @@ def find_cloud(ip_port=None,
tries = 0
# we could just take a single node's word on the complete cloud, but this
# two layer try is no big deal and gives some checking robustness when a bad cloud exists
hdfsSetup = (hdfs_version, hdfs_config, hdfs_name_node)
for n1, possMember in enumerate(possMembers):
tries += 1
if possMember not in probes:
probes.add(possMember)
members2 = probe_node(possMember, h2oNodes, expectedSize)
members2 = probe_node(possMember, h2oNodes, expectedSize, hdfsSetup)
for n2, member2 in enumerate(members2):
tries += 1
if member2 not in probes:
probes.add(member2)
probe_node(member2, h2oNodes)
probe_node(member2, h2oNodes, expectedSize, hdfsSetup)

print "\nDid %s tries" % tries
print "len(probe):", len(probes)
Expand Down
1 change: 0 additions & 1 deletion py/h2o_gbm.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,6 @@ def simpleCheckGBMView(node=None, gbmv=None, noPrint=False, **kwargs):

sample_rate = kwargs.get('sample_rate', None)
validation = kwargs.get('validation', None)
print "kevin:", sample_rate, validation
if (sample_rate==1 and not validation):
pass
elif (totalScores<=0 or totalScores>5e9):
Expand Down
1 change: 0 additions & 1 deletion py/h2o_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ def __init__(self,
if self.h2o_addr:
self.http_addr = self.h2o_addr
else:
print "kevin", h2o_args.python_cmd_ip
self.http_addr = h2o_args.python_cmd_ip

if h2o_args.network_from_cmd_line:
Expand Down

0 comments on commit 6e97168

Please sign in to comment.