Merge branch 'master' of https://github.com/0xdata/h2o

R/h2o-package/R/Classes.R

@@ -166,10 +166,10 @@ setMethod("[", "H2OParsedData", function(x, i, j, ..., drop = TRUE) {
   if(missing(i) && missing(j)) return(x)
   if(missing(i) && !missing(j)) {
     if(is.character(j)) return(do.call("$", c(x, j)))
-    expr = paste(x@key, "[", j-1, "]", sep="")
+    expr = paste(h2o.__escape(x@key), "[", j-1, "]", sep="")
   } else {
     if(class(i) == "H2OLogicalData") {
-      opt = paste(x@key, i@key, sep=",")
+      opt = paste(h2o.__escape(x@key), h2o.__escape(i@key), sep=",")
       if(missing(j))
         expr = paste("filter(", opt, ")", sep="")
       else if(is.character(j))
@@ -180,7 +180,7 @@ setMethod("[", "H2OParsedData", function(x, i, j, ..., drop = TRUE) {
     }
     else if(is.numeric(i)) {
       start = min(i); i_off = i - start + 1;
-      opt = paste(x@key, start-1, max(i_off), sep=",")
+      opt = paste(h2o.__escape(x@key), start-1, max(i_off), sep=",")
       if(missing(j))
         expr = paste("slice(", opt, ")", sep="")
       else if(is.character(j))
@@ -199,7 +199,7 @@ setMethod("$", "H2OParsedData", function(x, name) {
     print("Column", as.character(name), "not present in expression"); return(NULL)
   } else {
     # x[match(name, myNames)]
-    expr = paste(x@key, "$", name, sep="")
+    expr = paste(h2o.__escape(x@key), "$", name, sep="")
     res = h2o.__exec(x@h2o, expr)
     new("H2OParsedData", h2o=x@h2o, key=res)
   }
@@ -281,8 +281,8 @@ setMethod("head", "H2OParsedData", function(x, n = 6L, ...) {
 setMethod("tail", "H2OParsedData", function(x, n = 6L, ...) {
   if(n == 0 || !is.numeric(n)) stop("n must be a non-zero integer")
   n = round(n)
-  if(n > 0) opt = paste(x@key, nrow(x)-n, sep=",")
-  else opt = paste(x@key, abs(n), sep=",")
+  if(n > 0) opt = paste(h2o.__escape(x@key), nrow(x)-n, sep=",")
+  else opt = paste(h2o.__escape(x@key), abs(n), sep=",")
   res = h2o.__exec(x@h2o, paste("slice(", opt, ")", sep=""))
   as.data.frame(new("H2OParsedData", h2o=x@h2o, key=res))
 })
@@ -295,8 +295,8 @@ setMethod("plot", "H2OPCAModel", function(x, y, ...) {
 setGeneric("h2o.factor", function(data, col) { standardGeneric("h2o.factor") })
 setMethod("h2o.factor", signature(data="H2OParsedData", col="numeric"),
    function(data, col) {
-      newCol = paste("factor(", data@key, "[", col, "])", sep="")
-      expr = paste("colSwap(", data@key, ",", col, ",", newCol, ")", sep="")
+      newCol = paste("factor(", h2o.__escape(data@key), "[", col, "])", sep="")
+      expr = paste("colSwap(", h2o.__escape(data@key), ",", col, ",", newCol, ")", sep="")
       res = h2o.__exec_dest_key(data@h2o, expr, destKey=data@key)
       data
 })

R/h2o-package/R/Internal.R

@@ -139,8 +139,8 @@ h2o.__exec_dest_key <- function(client, expr, destKey) {
 h2o.__operator <- function(op, x, y) {
   if(!((ncol(x) == 1 || class(x) == "numeric") && (ncol(y) == 1 || class(y) == "numeric")))
     stop("Can only operate on single column vectors")
-  LHS = ifelse(class(x) == "H2OParsedData", x@key, x)
-  RHS = ifelse(class(y) == "H2OParsedData", y@key, y)
+  LHS = ifelse(class(x) == "H2OParsedData", h2o.__escape(x@key), x)
+  RHS = ifelse(class(y) == "H2OParsedData", h2o.__escape(y@key), y)
   expr = paste(LHS, op, RHS)
   if(class(x) == "H2OParsedData") myClient = x@h2o
   else myClient = y@h2o
@@ -151,9 +151,13 @@ h2o.__operator <- function(op, x, y) {
     new("H2OParsedData", h2o=myClient, key=res)
 }
 
+h2o.__escape <- function(key) {
+  paste("|", key, "|", sep="")
+}
+
 h2o.__func <- function(fname, x, type) {
   if(ncol(x) != 1) stop("Can only operate on single column vectors")
-  expr = paste(fname, "(", x@key, ")", sep="")
+  expr = paste(fname, "(", h2o.__escape(x@key), ")", sep="")
   res = h2o.__exec(x@h2o, expr)
   res = h2o.__remoteSend(x@h2o, h2o.__PAGE_INSPECT, key=res)
 

py/h2o.py

@@ -1,4 +1,4 @@
-import time, os, json, signal, tempfile, shutil, datetime, inspect, threading, getpass
+import time, os, stat, json, signal, tempfile, shutil, datetime, inspect, threading, getpass
 import requests, psutil, argparse, sys, unittest, glob
 import h2o_browse as h2b, h2o_perf, h2o_util, h2o_cmd, h2o_os_util
 import h2o_sandbox
@@ -261,13 +261,22 @@ def clean_sandbox_stdout_stderr():
             os.remove(f)
 
 def tmp_file(prefix='', suffix=''):
-    return tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=LOG_DIR)
+    fd, path = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=LOG_DIR)
+    # make sure the file now exists
+    # os.open(path, 'a').close()
+    # give everyone permission to read it (jenkins running as 
+    # 0xcustomer needs to archive as jenkins
+    permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+    os.chmod(path, permissions)
+    return (fd, path)
 
 def tmp_dir(prefix='', suffix=''):
     return tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=LOG_DIR)
 
 def log(cmd, comment=None):
-    with open(LOG_DIR + '/commands.log', 'a') as f:
+    filename = LOG_DIR + '/commands.log'
+    # everyone can read
+    with open(filename, 'a') as f:
         f.write(str(datetime.datetime.now()) + ' -- ')
         # what got sent to h2o
         # f.write(cmd)
@@ -280,6 +289,9 @@ def log(cmd, comment=None):
             f.write("\n")
         elif comment: # for comment-only
             f.write(comment + "\n")
+    # jenkins runs as 0xcustomer, and the file wants to be archived by jenkins who isn't in his group
+    permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+    os.chmod(filename, permissions)
 
 def make_syn_dir():
     SYNDATASETS_DIR = './syn_datasets'
@@ -297,6 +309,7 @@ def spawn_cmd(name, cmd, capture_output=True, **kwargs):
     if capture_output:
         outfd, outpath = tmp_file(name + '.stdout.', '.log')
         errfd, errpath = tmp_file(name + '.stderr.', '.log')
+        # everyone can read
         ps = psutil.Popen(cmd, stdin=None, stdout=outfd, stderr=errfd, **kwargs)
     else:
         outpath = '<stdout>'

py/testdir_release/create_Renviron_Rprofile.sh

@@ -0,0 +1,12 @@
+
+echo "Removing any existing .Renviron and .Rprofile and recreating"
+rm -f ~/.Renviron
+rm -f ~/.Rprofile
+# Set CRAN mirror to a default location
+echo "options(repos = \"http://cran.stat.ucla.edu\")" > ~/.Rprofile
+# Needed so the h2oWrapper doesn't try installing to a site library
+# normally first in .libPaths() in R, and only writeable by root.
+# .libPaths() will show this first now
+echo "R_LIBS_USER=\"~/.Rlibrary\"" > ~/.Renviron
+# I don't rm it if it already exits
+mkdir -p ~/.Rlibrary

py/testdir_release/pytest_config-kevin.json

@@ -19,7 +19,7 @@
     "use_flatfile": true,
 
     "ip": [
-        "192.168.0.37"
+        "192.168.1.80"
     ]
 }
 

py/testdir_release/runner_sh2junit.sh

@@ -16,7 +16,6 @@ set -e
 rm -f test.*xml
 rm -f TEST*xml
 
-
 # This gets the h2o.jar
 source ./runner_setup.sh
 
@@ -82,13 +81,18 @@ mySetup() {
     # then make the R_LIB_USERS dir
     which R
     R --version
-    rm -f ~/.Renviron
-    rm -f ~/.Rprofile
-    # Set CRAN mirror to a default location
-    echo "options(repos = \"http://cran.stat.ucla.edu\")" > ~/.Rprofile
-    echo "R_LIBS_USER=\"~/.Rlibrary\"" > ~/.Renviron
-    rm -f -r ~/.Rlibrary
-    mkdir -p ~/.Rlibrary
+
+    # don't always remove..other users may have stuff he doesn't want to re-install
+    if [[ $USER == "jenkins" ]]
+    then 
+        # Set CRAN mirror to a default location
+        rm -f ~/.Renviron
+        rm -f ~/.Rprofile
+        echo "options(repos = \"http://cran.stat.ucla.edu\")" > ~/.Rprofile
+        echo "R_LIBS_USER=\"~/.Rlibrary\"" > ~/.Renviron
+        rm -f -r ~/.Rlibrary
+        mkdir -p ~/.Rlibrary
+    fi
 
     echo ".libPaths()" > /tmp/libPaths.cmd
     cmd="R -f /tmp/libPaths.cmd --args $CLOUD_IP:$CLOUD_PORT"
@@ -150,7 +154,7 @@ mySetup 'libPaths'
 myR 'runit_RF.R' 35
 myR 'runit_PCA.R' 35
 myR 'runit_GLM.R' 35
-myR 'runit_GBM.R' 35
+myR 'runit_GBM.R' 300
 # If this one fails, fail this script so the bash dies 
 # We don't want to hang waiting for the cloud to terminate.
 # produces xml too!