Merge branch 'master' of https://github.com/0xdata/h2o

Author: spennihana

R/h2oRClient-package/R/Internal.R

@@ -52,7 +52,11 @@ h2o.__remoteSend <- function(client, page, ...) {
   #TODO (Spencer): Create "commands.log" using: list(...)
   # Sends the given arguments as URL arguments to the given page on the specified server
   url = paste("http://", ip, ":", port, "/", page, sep="")
-  temp = postForm(url, style = "POST", ...)
+  # temp = postForm(url, style = "POST", ...)
+  if(length(list(...)) == 0)
+    temp = getURLContent(url)
+  else
+    temp = getForm(url, ..., .checkParams = FALSE)   # Some H2O params overlap with Curl params
   # after = gsub("NaN", "\"NaN\"", temp[1])
   after = gsub("\\\\\\\"NaN\\\\\\\"", "NaN", temp[1])    # TODO: Don't escape NaN in the JSON!
   after = gsub("NaN", "\"NaN\"", after)

bench/BMscripts/glm2Bench.py

@@ -1,5 +1,5 @@
 #GLM2 bench
-import os, sys, time, csv
+import os, sys, time, csv, re, requests
 sys.path.append('../py/')
 sys.path.extend(['.','..'])
 import h2o_cmd, h2o, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_rf, h2o_jobs
@@ -108,15 +108,27 @@ def doGLM2(f, folderPath, family, lambda_, alpha, nfolds, y, x, testFilehex, row
         row.update( {'AIC'          : glmView['glm_model']['validation']['aic'],
                      'nIterations'  : glmView['glm_model']['iteration'],
                      'nPredictors'  : len(glmView['glm_model']['beta']),
-                     'AverageError' : glmView['glm_model']['validation']['avg_err'],
+                     #'AverageError' : glmView['glm_model']['validation']['avg_err'],
                     })
         if family == "binomial":
+            #Scrape html of 2/glmmodelview to get best threshold,
+            #then, multiply by 100 and cast to int...
+            #then ask for the coresponding CM from _cms inside glmView
+            url     = 'http://%s:%d/2/GLMModelView.html?_modelKey=%s' % (h2o.nodes[0].http_addr, 55555, 'GLM('+f+')')
+            r       = requests.get(url).text
+            p1      = re.compile('threshold[:<>/a-z]*[0-9]\.[0-9]*')
+            p2      = re.compile('[0-9]\.[0-9]*')
+            best    = int(float(p2.search(p1.search(text).group()).group()) * 100)
+            best_cm = glmView['glm_model']['validation']['_cms'][best]['_arr']
+            avg_err = (best_cm[0][1] + best_cm[1][0]) / (sum([i for sublist in best_cm for i in sublist]))
             row.update( {#'scoreTime'          : scoreTime,
                          'AUC'                : glmView['glm_model']['validation']['auc'],
+                         'AverageError'       : avg_err,
                         })
         else:
             row.update( {#'scoreTime'          : scoreTime,
                          'AUC'                : 'NA',
+                         'AverageError'       : glmView['glm_model']['validation']['avg_err'],
                         })
         csvWrt.writerow(row)
     finally:

py/h2o.py

@@ -1190,36 +1190,6 @@ def kmeans(self, key, key2=None,
             time.sleep(5)
         return a
 
-    def kmeans_grid(self, key, key2=None,
-        timeoutSecs=300, retryDelaySecs=0.2, initialDelaySecs=None, pollTimeoutSecs=180,
-        **kwargs):
-        # defaults
-        params_dict = {
-            'initialization': 'Furthest',
-            'k': 1,
-            'max_iter': 10,
-            'source_key': key,
-            'destination_key': 'python_KMeans_Grid_destination.hex',
-            }
-        browseAlso = kwargs.get('browseAlso', False)
-        params_dict.update(kwargs)
-        print "\nKMeansGrid params list:", params_dict
-        a = self.__do_json_request('KMeansGrid.json', timeout=timeoutSecs, params=params_dict)
-
-        # Check that the response has the right Progress url it's going to steer us to.
-        if a['response']['redirect_request']!='Progress':
-            print dump_json(a)
-            raise Exception('H2O kmeans_grid redirect is not Progress. KMeans json response precedes.')
-        a = self.poll_url(a, timeoutSecs=timeoutSecs, retryDelaySecs=retryDelaySecs,
-            initialDelaySecs=initialDelaySecs, pollTimeoutSecs=pollTimeoutSecs)
-        verboseprint("\nKMeansGrid result:", dump_json(a))
-
-        if (browseAlso | browse_json):
-            print "Redoing the KMeansGrid through the browser, no results saved though"
-            h2b.browseJsonHistoryAsUrlLastMatch('KMeansGrid')
-            time.sleep(5)
-        return a
-
     # params:
     # header=1,
     # header_from_file

py/h2o_cmd.py

@@ -59,13 +59,6 @@ def runKMeans(node=None, parseResult=None, timeoutSecs=20, retryDelaySecs=2, noP
     if not node: node = h2o.nodes[0]
     return node.kmeans(parseResult['destination_key'], None, timeoutSecs, retryDelaySecs, noPoll=noPoll, **kwargs)
 
-def runKMeansGrid(node=None, parseResult=None,
-        timeoutSecs=60, retryDelaySecs=2, noise=None, **kwargs):
-    if not parseResult: raise Exception('No parseResult for KMeansGrid')
-    if not node: node = h2o.nodes[0]
-    # no such thing as KMeansGridView..don't use retryDelaySecs
-    return node.kmeans_grid(parseResult['destination_key'], timeoutSecs, **kwargs)
-
 def runGLM(node=None, parseResult=None, 
         timeoutSecs=20, retryDelaySecs=2, noise=None, **kwargs):
     if not parseResult: raise Exception('No parseResult for GLM')

py/testdir_multi_jvm/test_KMeansGrid_basic.py

@@ -53,16 +53,17 @@ def test_KMeansGrid_basic(self):
                 'initialization': None,
                 'seed': 3923021996079663354, 
                 'normalize': 0, 
-                'max_iter': 10
+                'max_iter': "c(2,5,10)"
             }
+    
             for trial in range(3):
                 kwargs = params.copy()
-
+                h2o.beta_features = True
                 start = time.time()
-                kmeans = h2o_cmd.runKMeansGrid(parseResult=parseResult, \
+                kmeans = h2o_cmd.runKMeans(parseResult=parseResult, \
                     timeoutSecs=timeoutSecs, retryDelaySecs=2, pollTimeoutSecs=60, **kwargs)
                 elapsed = time.time() - start
-                print "kmeans grid end on ", csvPathname, 'took', elapsed, 'seconds.', \
+                print "kmeans (with grid) end on ", csvPathname, 'took', elapsed, 'seconds.', \
                     "%d pct. of timeout" % ((elapsed/timeoutSecs) * 100)
                 h2o_kmeans.simpleCheckKMeans(self, kmeans, **kwargs)
 

py/testdir_multi_jvm/test_KMeansGrid_params_rand2.py

@@ -60,16 +60,17 @@ def test_KMeans_params_rand2(self):
                 "    num_cols:", "{:,}".format(inspect['num_cols'])
 
             paramDict = define_params(SEED)
+            h2o.beta_features = True # no grid for VA
             for trial in range(3):
                 # default
-                params = {'k': 1 }
+                params = {'k': 'c(1,2,3)' }
                 # 'destination_key': csvFilename + "_" + str(trial) + '.hex'}
 
                 h2o_kmeans.pickRandKMeansParams(paramDict, params)
                 kwargs = params.copy()
-
+        
                 start = time.time()
-                kmeans = h2o_cmd.runKMeansGrid(parseResult=parseResult, \
+                kmeans = h2o_cmd.runKMeans(parseResult=parseResult, \
                     timeoutSecs=timeoutSecs, retryDelaySecs=2, pollTimeoutSecs=60, **kwargs)
                 elapsed = time.time() - start
                 print "kmeans grid end on ", csvPathname, 'took', elapsed, 'seconds.', \

py/testdir_single_jvm/test_GBM_many_cols.py

@@ -35,7 +35,7 @@ def setUpClass(cls):
             # fails
             # h2o.build_cloud(1,java_heap_MB=100, enable_benchmark_log=True)
             # 400 fails
-            h2o.build_cloud(1,java_heap_GB=8, enable_benchmark_log=True)
+            h2o.build_cloud(1,java_heap_GB=2, enable_benchmark_log=True)
         else:
             h2o_hosts.build_cloud_with_hosts(enable_benchmark_log=True)
 

py/testdir_single_jvm/test_enum_multi_permission.py

@@ -76,7 +76,7 @@ def test_cols_enum_multi_import(self):
             # DON"T get redirected to S3! (EC2 hack in config, remember!)
             # use it at the node level directly (because we gen'ed the files.
             # use regex. the only files in the dir will be the ones we just created with  *fileN* match
-            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='put',
+            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='local',
                 exclude=None, header=1, timeoutSecs=timeoutSecs)
             print "parseResult['destination_key']: " + parseResult['destination_key']
             print 'parse time:', parseResult['response']['time']
@@ -104,23 +104,23 @@ def test_cols_enum_multi_import(self):
             print "checking os.chmod and parse"
             # os.chmod(badPathname, stat.S_IRWXU | stat.S_IRWXO)
             # always have to re-import because source key is deleted by h2o
-            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='put',
+            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='local',
                 exclude=None, header=1, timeoutSecs=timeoutSecs)
             print "parseResult['destination_key']: " + parseResult['destination_key']
             inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
             h2o_cmd.infoFromInspect(inspect, csvPathname)
 
             print "write by owner, only, and parse"
             os.chmod(badPathname, stat.S_IWRITE)
-            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='put',
+            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='local',
                 exclude=None, header=1, timeoutSecs=timeoutSecs)
             inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
             h2o_cmd.infoFromInspect(inspect, csvPathname)
 
             print "execute by owner, only, and parse"
             os.chmod(badPathname, stat.S_IEXEC)
             h2o.nodes[0].import_files(SYNDATASETS_DIR)
-            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='put',
+            parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='local',
                 exclude=None, header=1, timeoutSecs=timeoutSecs)
             inspect = h2o_cmd.runInspect(None, parseResult['destination_key'])
             h2o_cmd.infoFromInspect(inspect, csvPathname)
@@ -136,11 +136,11 @@ def test_cols_enum_multi_import(self):
 
                 print "parsing after one bad uid"
                 os.chown(badPathname, badUid, origGid)
-                parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='put',
+                parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='local',
                     exclude=None, header=1, timeoutSecs=timeoutSecs)
                 print "parsing after one bad gid"
                 os.chown(badPathname, origUid, badGid)
-                parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='put',
+                parseResult = h2i.import_parse(path=SYNDATASETS_DIR + '/*'+rowxcol+'*', schema='local',
                     exclude=None, header=1, timeoutSecs=timeoutSecs)
 
                 os.chown(badPathname, origUid, origGid)

src/main/java/hex/pca/PCAScore.java

@@ -90,7 +90,7 @@ public static String link(Key modelKey, String content) {
   }
 
   public static String link(String key_param, Key k, String content) {
-    RString rs = new RString("<a href='PCAScore.query?%key_param=%$key'>%content</a>");
+    RString rs = new RString("<a href='2/PCAScore.query?%key_param=%$key'>%content</a>");
     rs.replace("key_param", key_param);
     rs.replace("key", k.toString());
     rs.replace("content", content);