Merge branch 'master' of github.com:0xdata/h2o

tomasnykodym · tomasnykodym · commit 114856b3542d · 2014-07-25T10:41:52.000-07:00
diff --git a/R/h2o-package/R/Classes.R b/R/h2o-package/R/Classes.R
@@ -498,6 +498,12 @@ h2o.ddply <- function (.data, .variables, .fun = NULL, ..., .progress = 'none')
   if( any(bad) ) stop( sprintf('can\'t recognize .variables %s', paste(vars[bad], sep=',')) )
   
   fun_name <- mm[[ '.fun' ]]
+
+  if(identical(as.list(substitute(.fun))[[1]], quote(`function`))) {
+    h2o.addFunction(.data@h2o, .fun, "anonymous")
+    fun_name <- "anonymous"
+  }
+
   exec_cmd <- sprintf('ddply(%s,c(%s),%s)', .data@key, paste(idx, collapse=','), as.character(fun_name))
   res <- .h2o.__exec2(.data@h2o, exec_cmd)
   .h2o.exec2(res$dest_key, h2o = .data@h2o, res$dest_key)
@@ -520,7 +526,7 @@ h2o.addFunction <- function(object, fun, name){
     if( class(name) != 'character' ) stop('name must be a name')
     fun_name <- name
   } else {
-    fun_name <- match.call()[['fun']]
+    fun_name <- "anonymous" #fun_name <- match.call()[['fun']]
   }
   src <- paste(deparse(fun), collapse='\n')
   exec_cmd <- sprintf('%s <- %s', as.character(fun_name), src)
@@ -1108,15 +1114,10 @@ head.H2OParsedData <- function(x, n = 6L, ...) {
   stopifnot(length(n) == 1L)
   n <- ifelse(n < 0L, max(numRows + n, 0L), min(n, numRows))
   if(n == 0) return(data.frame())
-  
-  x.slice = as.data.frame(x[seq_len(n),])
-#   if(ncol(x) > .MAX_INSPECT_COL_VIEW)
-#     warning(x@key, " has greater than ", .MAX_INSPECT_COL_VIEW, " columns. This may take awhile...")
-#   res = .h2o.__remoteSend(x@h2o, .h2o.__HACK_LEVELS2, source = x@key, max_ncols = .Machine$integer.max)
-#   for(i in 1:ncol(x)) {
-#     if(!is.null(res$levels[[i]]))
-#       x.slice[,i] <- factor(x.slice[,i], levels = res$levels[[i]])
-#   }
+
+  tmp_head <- x[seq_len(n),]
+  x.slice = as.data.frame(tmp_head)
+  h2o.rm(tmp_head@h2o, tmp_head@key)
   return(x.slice)
 }
 
@@ -1127,16 +1128,10 @@ tail.H2OParsedData <- function(x, n = 6L, ...) {
   if(n == 0) return(data.frame())
   
   idx <- seq.int(to = nrx, length.out = n)
-  x.slice <- as.data.frame(x[idx,])
+  tmp_tail <- x[idx,]
+  x.slice <- as.data.frame(tmp_tail)
+  h2o.rm(tmp_tail@h2o, tmp_tail@key)
   rownames(x.slice) <- idx
-  
-#   if(ncol(x) > .MAX_INSPECT_COL_VIEW)
-#     warning(x@key, " has greater than ", .MAX_INSPECT_COL_VIEW, " columns. This may take awhile...")
-#   res = .h2o.__remoteSend(x@h2o, .h2o.__HACK_LEVELS2, source = x@key, max_ncols = .Machine$integer.max)
-#   for(i in 1:ncol(x)) {
-#     if(!is.null(res$levels[[i]]))
-#       x.slice[,i] <- factor(x.slice[,i], levels = res$levels[[i]])
-#   }
   return(x.slice)
 }
 
@@ -1254,34 +1249,83 @@ screeplot.H2OPCAModel <- function(x, npcs = min(10, length(x@model$sdev)), type
   as.logical(.h2o.__unop2("canBeCoercedToLogical", vec))
 }
 
-setMethod("ifelse", signature(test="H2OParsedData", yes="ANY", no="ANY"), function(test, yes, no) {
-  if(!(is.numeric(yes) || class(yes) == "H2OParsedData") || !(is.numeric(no) || class(no) == "H2OParsedData"))
-    stop("Unimplemented")
-  if(!test@logic && !.canBeCoercedToLogical(test)) stop(test@key, " is not a H2O logical data type")
-  .h2o.__multop2("ifelse", test, yes, no)
-})
-
-setMethod("ifelse", signature(test="logical", yes="H2OParsedData", no="numeric"), function(test, yes, no) {
-  if(length(test) > 1) stop("test must be a single logical value")
-  .h2o.__multop2("ifelse", as.numeric(test), yes, no)
-})
-
-setMethod("ifelse", signature(test="logical", yes="numeric", no="H2OParsedData"), function(test, yes, no) {
-  if(length(test) > 1) stop("test must be a single logical value")
-  .h2o.__multop2("ifelse", as.numeric(test), yes, no)
-})
+.check.ifelse.conditions <-
+function(test, yes, no, type) {
+ if (type == "test") {
+   return(class(test) == "H2OParsedData"
+            && (is.numeric(yes) || class(yes) == "H2OParsedData" || is.logical(yes))
+            && (is.numeric(no) || class(no) == "H2OParsedData" || is.logical(no))
+            && (test@logic || .canBeCoercedToLogical(test)))
+ }
+}
 
-setMethod("ifelse", signature(test="logical", yes="H2OParsedData", no="H2OParsedData"), function(test, yes, no) {
-  if(length(test) > 1) stop("test must be a single logical value")
-  .h2o.__multop2("ifelse", as.numeric(test), yes, no)
-})
+ifelse<-
+function (test, yes, no)
+{
+    if (.check.ifelse.conditions(test, yes, no, "test")) {
+      if (is.logical(yes)) yes <- as.numeric(yes)
+      if (is.logical(no)) no <- as.numeric(no)
+      return(.h2o.__multop2("ifelse", test, yes, no))
+
+    } else if ( class(yes) == "H2OParsedData" && class(test) == "logical") {
+      if (is.logical(yes)) yes <- as.numeric(yes)
+      if (is.logical(no)) no <- as.numeric(no)
+      return(.h2o.__multop2("ifelse", as.numeric(test), yes, no))
+
+    } else if (class(no) == "H2OParsedData" && class(test) == "logical") {
+      if (is.logical(yes)) yes <- as.numeric(yes)
+      if (is.logical(no)) no <- as.numeric(no)
+      return(.h2o.__multop2("ifelse", as.numeric(test), yes, no))
+    }
+    if (is.atomic(test))
+        storage.mode(test) <- "logical"
+    else test <- if (isS4(test))
+        as(test, "logical")
+    else as.logical(test)
+    ans <- test
+    ok <- !(nas <- is.na(test))
+    if (any(test[ok]))
+        ans[test & ok] <- rep(yes, length.out = length(ans))[test &
+            ok]
+    if (any(!test[ok]))
+        ans[!test & ok] <- rep(no, length.out = length(ans))[!test &
+            ok]
+    ans[nas] <- NA
+    ans
+}
 
-setMethod("levels", "H2OParsedData", function(x) {
-  # if(ncol(x) != 1) return(NULL)
-  if(ncol(x) != 1) stop("Can only retrieve levels of one column.")
-  res = .h2o.__remoteSend(x@h2o, .h2o.__HACK_LEVELS2, source = x@key, max_ncols = .Machine$integer.max)
-  res$levels[[1]]
-})
+#setMethod("ifelse", signature(test="H2OParsedData", yes="ANY", no="ANY"), function(test, yes, no) {
+#  if(!(is.numeric(yes) || class(yes) == "H2OParsedData") || !(is.numeric(no) || class(no) == "H2OParsedData"))
+#    stop("Unimplemented")
+#  if(!test@logic && !.canBeCoercedToLogical(test)) stop(test@key, " is not a H2O logical data type")
+#  h2o.exec(ifelse(test, yes, no))
+##  .h2o.__multop2("ifelse", eval(test), yes, no)
+#})
+##
+#setMethod("ifelse", signature(test="logical", yes="H2OParsedData", no="ANY"), function(test, yes, no) {
+#  if(length(test) > 1) stop("test must be a single logical value")
+#  h2o.exec(ifelse(test, yes, no))
+##  .h2o.__multop2("ifelse", as.numeric(test), eval(yes), no)
+#})
+#
+#setMethod("ifelse", signature(test="logical", yes="ANY", no="H2OParsedData"), function(test, yes, no) {
+#  if(length(test) > 1) stop("test must be a single logical value")
+#  h2o.exec(ifelse(test, yes, no))
+##  .h2o.__multop2("ifelse", as.numeric(test), yes, eval(no))
+#})
+#
+#setMethod("ifelse", signature(test="logical", yes="H2OParsedData", no="H2OParsedData"), function(test, yes, no) {
+#  if(length(test) > 1) stop("test must be a single logical value")
+#  h2o.exec(ifelse(test, yes, no))
+##  .h2o.__multop2("ifelse", as.numeric(test), eval(yes), eval(no))
+#})
+#
+#setMethod("levels", "H2OParsedData", function(x) {
+#  # if(ncol(x) != 1) return(NULL)
+#  if(ncol(x) != 1) stop("Can only retrieve levels of one column.")
+#  res = .h2o.__remoteSend(x@h2o, .h2o.__HACK_LEVELS2, source = x@key, max_ncols = .Machine$integer.max)
+#  res$levels[[1]]
+#})
 
 #----------------------------- Work in Progress -------------------------------#
 # TODO: Need to change ... to environment variables and pass to substitute method,
diff --git a/R/tests/testdir_jira/runit_pub_486_nested_ifelse.R b/R/tests/testdir_jira/runit_pub_486_nested_ifelse.R
@@ -8,6 +8,13 @@ test.nested_ifelse <- function(conn) {
   e <- ifelse(a == b, b, ifelse(b == b, ifelse(a == d, 3.1415, 0), a))
   print(e)
 
+  r.hex <- as.h2o(conn, iris[,1:4])
+  f <- ifelse( a == 1, !c(3), 1.23)
+  g <- ifelse( b == 0, !c(!3), 1.23 <= 2.34)
+
+  h2o.exec( f <- ifelse( a == 1, !c(1), 1.23))
+  h2o.exec(g <- ifelse( b == 0, !c(!3), 1.23 <= 2.34)) 
+
   testEnd()
 }
 
diff --git a/R/tests/testdir_jira/runit_pub_532_negative_exponent.R b/R/tests/testdir_jira/runit_pub_532_negative_exponent.R
@@ -14,11 +14,27 @@ h2o.exec(covtype.hex$C55^2)
 h2o.exec(covtype.hex$C55^(-2))
 
 # currently fails:
-h2o.exec(covtype.hex$C55^-2)
+val0 <- h2o.exec(covtype.hex$C55^-2)
 h2o.exec(covtype.hex$C55^ -2)
 h2o.exec(covtype.hex$C55 ^-2)
 h2o.exec(covtype.hex$C55 ^-2 )
 
+
+#w/o h2o.exec:
+val <- covtype.hex$C55^-2
+covtype.hex$C55^ -2
+covtype.hex$C55 ^-2
+covtype.hex$C55 ^-2 
+
+print(val0)
+
+print("================")
+
+print(val)
+
+print(tail(val))
+
+
 testEnd()
 
 }
diff --git a/R/tests/testdir_jira/runit_pub_575_ifelse_with_assign.R b/R/tests/testdir_jira/runit_pub_575_ifelse_with_assign.R
@@ -0,0 +1,22 @@
+setwd(normalizePath(dirname(R.utils::commandArgs(asValues=TRUE)$"f")))
+source('../findNSourceUtils.R')
+
+test.pub_575 <- function(localH2O) {
+
+covtype.hex <- h2o.importFile(localH2O, normalizePath(locate("smalldata/covtype/covtype.20k.data")), "cov")
+
+hex <- covtype.hex
+
+print(ifelse(TRUE, hex, hex[,1] <- hex[,1] + 1))
+print(h2o.exec(ifelse(1, hex, hex[,1] <- hex[,1] + 1)))
+
+
+#ensure that base ifelse is not broken
+print(ifelse(TRUE, iris, iris[,1] <- iris[,1] + 1))
+
+testEnd()
+
+}
+
+doTest("PUB-575 ifelse with embedded assignment", test.pub_575)
+
diff --git a/R/tests/testdir_jira/runit_pub_696_rep_len_cbind_large.R b/R/tests/testdir_jira/runit_pub_696_rep_len_cbind_large.R
@@ -0,0 +1,20 @@
+setwd(normalizePath(dirname(R.utils::commandArgs(asValues=TRUE)$"f")))
+source('../findNSourceUtils.R')
+
+test.pub.696 <- function(conn) {
+
+  a <- h2o.exec(a <- rep_len(0,1000000), h2o = conn)
+  b <- h2o.exec(b <- runif(a, -1), h2o = conn)
+  
+  d <- h2o.exec(d <- cbind(a,b), h2o = conn)
+
+  print(d)
+  print(dim(d))
+
+  a2 <- h2o.exec(a2 <- rep_len(0,1000000000))
+  b2 <- h2o.exec(b2 <- runif(a, -1))
+
+  testEnd()
+}
+
+doTest("cbind of something created by rep_len", test.pub.696 )
diff --git a/R/tests/testdir_jira/runit_pub_862_support_anonymous_functions_with_ddply.R b/R/tests/testdir_jira/runit_pub_862_support_anonymous_functions_with_ddply.R
diff --git a/R/tests/testdir_jira/runit_pub_863_disallow_addFunction_of_functions_that_already_exist.R b/R/tests/testdir_jira/runit_pub_863_disallow_addFunction_of_functions_that_already_exist.R
diff --git a/src/main/java/water/exec/ASTOp.java b/src/main/java/water/exec/ASTOp.java
@@ -161,10 +161,17 @@ public abstract class ASTOp extends AST {
     putPrefix(new ASTPrint ());
     putPrefix(new ASTLs    ());
   }
+  static private boolean isReserved(String fn) {
+    return UNI_INFIX_OPS.containsKey(fn) || BIN_INFIX_OPS.containsKey(fn) || PREFIX_OPS.containsKey(fn);
+  }
   static private void putUniInfix(ASTOp ast) { UNI_INFIX_OPS.put(ast.opStr(),ast); }
   static private void putBinInfix(ASTOp ast) { BIN_INFIX_OPS.put(ast.opStr(),ast); }
   static private void putPrefix  (ASTOp ast) {    PREFIX_OPS.put(ast.opStr(),ast); }
-  static         void putUDF     (ASTOp ast, String fn) {     UDF_OPS.put(fn,ast); }
+  static         void putUDF     (ASTOp ast, String fn) {
+    if (isReserved(fn)) throw new IllegalArgumentException("Trying to overload a reserved method: "+fn+". Must not overload a reserved method with a user-defined function.");
+    if (UDF_OPS.containsKey(fn)) removeUDF(fn);
+    UDF_OPS.put(fn,ast);
+  }
   static         void removeUDF  (String fn) { UDF_OPS.remove(fn); }
   static public ASTOp isOp(String id) {
     // This order matters. If used as a prefix OP, `+` and `-` are binary only.
@@ -447,7 +454,7 @@ class ASTCanBeCoercedToLogical extends ASTUniPrefixOp {
     Vec[] v = fr.vecs();
     for (Vec aV : v) {
       if (aV.isInt()) {
-        if (aV.min() == 0 && aV.max() == 1) {
+        if ((aV.min() == 0 && aV.max() == 1) || (aV.min() == 0 && aV.min() == aV.max()) || (aV.min() == 1 && aV.min() == aV.max())) {
           d = 1;
           break;
         }
diff --git a/src/main/java/water/exec/Exec2.java b/src/main/java/water/exec/Exec2.java
@@ -99,6 +99,8 @@ public boolean filter(H2O.KeyInfo k) {
     }
 
     // Some global constants
+    global.add(new ASTId(Type.DBL,"TRUE",0,global.size())); env.push(1.0);
+    global.add(new ASTId(Type.DBL,"FALSE",0,global.size())); env.push(0.0);
     global.add(new ASTId(Type.DBL,"T",0,global.size()));  env.push(1.0);
     global.add(new ASTId(Type.DBL,"F",0,global.size()));  env.push(0.0);
     global.add(new ASTId(Type.DBL,"NA",0,global.size()));  env.push(Double.NaN);
@@ -195,6 +197,7 @@ String isID() {
     if( isReserved(c) ) return null;
     // Fail on leading numeric
     if( isDigit(c) ) return null;
+    if (c == '^' && _buf[_x+1] == '-') return _str.substring(++_x -1, _x);
     _x++;                       // Accept parse of 1 char
 
     // If first char is letter, standard ID
