Merge branch 'master' of github.com:0xdata/h2o

Author: mmalohlava

R/examples/ClassifiersMSE.R

@@ -0,0 +1,43 @@
+## This script demonstrates how to compute MSE for classification problems
+## All the data manipulations happen in H2O
+## For simplicity, models are trained with default options, without validation data, and the training MSE is computed
+
+## Start H2O
+library(h2o)
+h2oServer = h2o.init(nthreads = -1)
+
+## Import data
+iris_hex <- as.h2o(h2oServer,iris) # this upload iris from R to H2O, use h2o.importFile() for big data!
+response <- 5
+predictors <- colnames(iris_hex[,-response])
+classes <- levels(iris_hex[,response])
+print(classes)
+
+## Compute actual per-row class probabilities (1 or 0)
+resp_hex <- iris_hex[,response]
+actual_hex <- resp_hex == classes[1]
+for (level in levels(resp_hex)[2:length(classes)]) {
+  actual_hex <- cbind(actual_hex, resp_hex==level)
+}
+summary(actual_hex)
+
+## Train H2O classifiers
+models <- list()
+models <- c(models, h2o.deeplearning(x = predictors, y = response, data = iris_hex))
+models <- c(models, h2o.randomForest(x = predictors, y = response, data = iris_hex))
+models <- c(models, h2o.randomForest(x = predictors, y = response, data = iris_hex, type="BigData"))
+models <- c(models, h2o.gbm(x = predictors, y = response, data = iris_hex))
+models <- c(models, h2o.naiveBayes(x = predictors, y = response, data = iris_hex))
+
+## Report Training MSE for all models
+for (model in models) {
+  ## Make predictions, extract probabilities
+  train_preds_hex <- h2o.predict(model, iris_hex)[,-1] # h2o.predict returns N+1 columns: label + N probabilities
+  
+  mse <- 0
+  for (i in 1:length(classes)) {
+    mse <- mse + mean((train_preds_hex[,i] - actual_hex[,i])^2)
+  }
+  print(paste0(model@key, " <---- MSE: ", mse))
+}
+

h2o-docs/source/faq/general.rst

@@ -4,19 +4,20 @@
 General Issues
 ==============
 
-## Downloading and Sending Logs
+Downloading and Sending Logs
+----------------------------
 
-1. From the drop-down **Admin** menu, select **Inspect Log** or go to <http://localhost:54321/LogView.html>.
+1. From the drop-down **Admin** menu, select **Inspect Log** or go to http://localhost:54321/LogView.html.
 2. On the **Log View** page, click the **Download Logs** button in the
 upper left hand corner to download your logs.
 
-.. Image:: Logsdownload.png
+.. Image:: Logsdownload.png 
    :width: 70%
 
 
-3. Email the logs to <[email protected]>  or <[email protected]> and include the following information: 
+3. Email the logs to [email protected] or [email protected] and include the following information: 
 	- H\ :sub:`2`\ O version number
-	- Your environment (laptop or server)
+	- Your environment (laptop or server) 
 	- Your operating system (Linux, Windows, OS X)
 	- Any other programs you are using with H\ :sub:`2`\ O (Hadoop, Cloudera, R)
 	- If you are using a cluster or other specific configuration

h2o-docs/source/faq/hadoop_related.rst

@@ -43,28 +43,35 @@ description of your Hadoop environment, including the Hadoop distribution and ve
 
 Common Hadoop Questions
 """"""""""""""""""""""""
-*What's the syntax for the file path of a data set sitting in hdfs?*
+**What's the syntax for the file path of a data set sitting in hdfs?**
 
 To locate an HDFS file, go to **Data > Import** and enter **hdfs://** in the **path** field. H\ :sub:`2`\ O automatically detects any HDFS paths. This is a good way to verify the path to your data set before importing through R or any other non-web API.
 
-*When interacting with an H2O cluster launched on multiple Hadoop nodes, is it necessary for R to be installed on all the data nodes?*
+---
+
+**When interacting with an H\ :sub:`2`\ O cluster launched on multiple Hadoop nodes, is it necessary for R to be installed on all the data nodes?**
 
 No - as long as the R instance can communicate with one of the nodes in the network, R can be installed on any of the nodes, or even on a local machine that will securely tunnel into the cluster.
 
-*Is it possible to launch the H2O cluster on Hadoop nodes using R’s h2o.init() command?*
+---
+
+**Is it possible to launch the H\ :sub:`2`\ O cluster on Hadoop nodes using R’s** `h2o.init()` **command?**
+
+No - follow the instructions in :ref:`Hadoop_Tutorial` and add the IP address to the `h2o.init()` function to connect to the cluster.
 
-No - follow the instructions in :ref:`Hadoop_Tutorial` and add the IP address to the **h2o.init()** function to connect to the cluster.
+---
 
-*What does "ERROR: Output directory hdfs://sandbox.hortonworks.com:8020/user/root/hdfsOutputDir already exists?" mean?*
+**What does** `"ERROR: Output directory hdfs://sandbox.hortonworks.com:8020/user/root/hdfsOutputDir already exists?"` **mean?**
 
 Each mapper task gets its own output directory in HDFS. To prevent overwriting multiple users' files, each mapper task must have a unique output directory name. Change
-the **-output hdfsOutputDir** argument to **-output hdfsOutputDir1** and the task should launch.
+the `-output hdfsOutputDir` argument to `-output hdfsOutputDir1` and the task should launch.
 
-*What should I do if H2O starts to launch but times out in 120 seconds?*
+---
 
+**What should I do if H\ :sub:`2`\ O  starts to launch but times out in 120 seconds?**
 
-1.YARN or MapReduce's configuration is not configured correctly. Enable launching for mapper tasks of specified memory sizes. If YARN only allows mapper tasks with a maximum memory size of 1g and the request requires 2g, then the request will timeout at the default of 120 seconds. Read `Configuration Setup <http://hortonworks.com/blog/how-to-plan-and-configure-yarn-in-hdp-2-0/>`_ to make sure your setup will run.
 
+1. YARN or MapReduce's configuration is not configured correctly. Enable launching for mapper tasks of specified memory sizes. If YARN only allows mapper tasks with a maximum memory size of 1g and the request requires 2g, then the request will timeout at the default of 120 seconds. Read `Configuration Setup <http://hortonworks.com/blog/how-to-plan-and-configure-yarn-in-hdp-2-0/>`_ to make sure your setup will run.
 
 2. The nodes are not communicating with each other. If you request a cluster of two nodes and the output shows a stall in reporting the other nodes and forming a cluster (as shown in the following example), check that the security settings for the network connection between the two nodes are not preventing the nodes from communicating with each other. You should also check to make sure that the flatfile that is generated and being passed has the correct home address; if there are multiple local IP addresses, this could be an issue.
 
@@ -94,17 +101,21 @@ the **-output hdfsOutputDir** argument to **-output hdfsOutputDir1** and the tas
         [Sending flatfile to node 10.115.5.25:54321]
     H2O node 10.115.57.45:54321 reports H2O cluster size 1
     H2O node 10.115.5.25:54321 reports H2O cluster size 1
+    
+---
 
-*What should I do if the H2O job launches but  terminates after 600 seconds?*
+**What should I do if the H2O job launches but  terminates after 600 seconds?**
 
 The likely cause is a driver mismatch - check to make sure the Hadoop distribution matches the driver jar file used to launch H\ :sub:`2`\ O. If your distribution is not currently
 available in the package, `email us <support@0xdata>`_ for a new driver file.
 
-*What should I do if I want to create a job with a bigger heap size but YARN doesn't launch and H2O times out?*
+---
 
-First, try the job again but with a smaller heap size (**-mapperXmx**) and a smaller number of nodes (**-nodes**) to verify that a small launch can proceed at all.
+**What should I do if I want to create a job with a bigger heap size but YARN doesn't launch and H\ :sub:`2`\ O times out?**
 
-If the cluster manager settings are configured for the default maximum memory size but the memory required for the request exceeds that amount, YARN will not launch and H2O will time out. 
+First, try the job again but with a smaller heap size (`-mapperXmx`) and a smaller number of nodes (`-nodes`) to verify that a small launch can proceed at all.
+
+If the cluster manager settings are configured for the default maximum memory size but the memory required for the request exceeds that amount, YARN will not launch and H\ :sub:`2`\ O  will time out. 
 If you have a default configuration, change the configuration settings in your cluster manager to enable launching of mapper tasks for specific memory sizes. Use the following formula to calculate the amount of memory required: 
 
 ::
@@ -134,51 +145,53 @@ Output from an H2O launch is shown below:
     mapreduce.map.memory.mb:     36864
 
 
-**mapreduce.map.memory.mb** must be less than the YARN memory configuration values for the launch to succeed.  See the examples below for how to change the memory configuration values for your version of Hadoop.
+`mapreduce.map.memory.mb` must be less than the YARN memory configuration values for the launch to succeed.  See the examples below for how to change the memory configuration values for your version of Hadoop.
 
 
 **For Cloudera, configure the settings in Cloudera Manager. Depending on how the cluster is configured, you may need to change the settings for more than one role group.**
 
-	1. Click **Configuration** and enter the following search term in quotes: **yarn.nodemanager.resource.memory-mb**.
+1. Click **Configuration** and enter the following search term in quotes: **yarn.nodemanager.resource.memory-mb**.
 
-	2. Enter the amount of memory (in GB) to allocate in the **Value** field. If more than one group is listed, change the values for all listed groups.
+2. Enter the amount of memory (in GB) to allocate in the **Value** field. If more than one group is listed, change the values for all listed groups.
 
 	.. image:: TroubleshootingHadoopClouderayarnnodemgr.png
 	   :width: 100 %	
 
-	3. Click the **Save Changes** button in the upper-right corner. 
-	4. Enter the following search term in quotes: **yarn.scheduler.maximum-allocation-mb**
-	5. Change the value, click the **Save Changes** button in the upper-right corner, and redeploy.
+3. Click the **Save Changes** button in the upper-right corner. 
+4. Enter the following search term in quotes: **yarn.scheduler.maximum-allocation-mb**
+5. Change the value, click the **Save Changes** button in the upper-right corner, and redeploy.
 
 	.. image:: TroubleshootingHadoopClouderayarnscheduler.png
 	   :width: 100%
 
 
-For Hortonworks, `configure <http://docs.hortonworks.com/HDPDocuments/Ambari-1.6.0.0/bk_Monitoring_Hadoop_Book/content/monitor-chap2-3-3_2x.html>`_ the settings in Ambari.
-	1. Select **YARN**, then click the **Configs** tab. 
-	2. Select the group. 
-	3. In the **Node Manager** section, enter the amount of memory (in MB) to allocate in the **yarn.nodemanager.resource.memory-mb** entry field. 
+**For Hortonworks,** `configure <http://docs.hortonworks.com/HDPDocuments/Ambari-1.6.0.0/bk_Monitoring_Hadoop_Book/content/monitor-chap2-3-3_2x.html>`_ **the settings in Ambari.**
+
+1. Select **YARN**, then click the **Configs** tab. 
+2. Select the group. 
+3. In the **Node Manager** section, enter the amount of memory (in MB) to allocate in the **yarn.nodemanager.resource.memory-mb** entry field. 
 
 	.. image:: TroubleshootingHadoopAmbariNodeMgr.png
 	  :width: 100 %
 
-	4. In the **Scheduler** section, enter the amount of memory (in MB)to allocate in the **yarn.scheduler.maximum-allocation-mb** entry field. 
+4. In the **Scheduler** section, enter the amount of memory (in MB)to allocate in the **yarn.scheduler.maximum-allocation-mb** entry field. 
 
 	.. image:: TroubleshootingHadoopAmbariyarnscheduler.png
 	  :width: 100 %
 
-	5. 	Click the **Save** button at the bottom of the page and redeploy the cluster. 
+5. 	Click the **Save** button at the bottom of the page and redeploy the cluster. 
 
 
-For MapR: 
-	1. Edit the **yarn-site.xml** file for the node running the ResourceManager. 
-	2. Change the values for the **yarn.nodemanager.resource.memory-mb** and **yarn.scheduler.maximum-allocation-mb** properties.
-	3. Restart the ResourceManager and redeploy the cluster. 
+**For MapR:**
+
+1. Edit the **yarn-site.xml** file for the node running the ResourceManager. 
+2. Change the values for the `yarn.nodemanager.resource.memory-mb` and `yarn.scheduler.maximum-allocation-mb` properties.
+3. Restart the ResourceManager and redeploy the cluster. 
 
 
 To verify the values were changed, check the values for the following properties:
 
-	* **<name>yarn.nodemanager.resource.memory-mb</name>**
-	* **<name>yarn.scheduler.maximum-allocation-mb</name>**
+	 - `<name>yarn.nodemanager.resource.memory-mb</name>`
+	 - `<name>yarn.scheduler.maximum-allocation-mb</name>`
 
- 
+---

src/main/java/hex/CoxPH.java

@@ -196,7 +196,7 @@ protected float[] score0(double[] data, float[] preds) {
       }
       for (int j = n_cats; j < n_data; ++j)
         numsHasNA |= Double.isNaN(data[j]);
-      if (numsHasNA || (catsHasNA && !catsAllNA) || (n_num == 0 && catsAllNA)) {
+      if (numsHasNA || (catsHasNA && !catsAllNA) || (n_num > 0 && catsAllNA)) {
         for (int i = 1; i <= 2 * n_time; ++i)
           preds[i] = Float.NaN;
       } else {

src/main/java/hex/FrameTask.java

@@ -136,6 +136,22 @@ public DataInfo(Frame fr, int hasResponses, boolean useAllFactorLvls, double []
 
     final int [][] _catLvls;
 
+    /**
+     * Apply data transformation on the given column.
+     *
+     * @param c - index into fully exponded vector
+     * @param v - value of the column to be transformed
+     * @return v transformed by the transformation (e.g. standardization) defined by this dataset for this column
+     */
+    public double applyTransform(int c, double v){
+      if(c >= _catOffsets[_catOffsets.length-1]) {
+        c -= _cats;
+        if (_normSub != null) v -= _normSub[c];
+        if (_normMul != null) v *= _normMul[c];
+      }
+      return v;
+    }
+
     /**
      * Prepare a Frame (with a single response) to be processed by the FrameTask
      * 1) Place response at the end
@@ -149,6 +165,9 @@ public DataInfo(Frame fr, int hasResponses, boolean useAllFactorLvls, double []
      * @return Frame to be used by FrameTask
      */
     public static Frame prepareFrame(Frame source, Vec response, int[] ignored_cols, boolean toEnum, boolean dropConstantCols, boolean dropNACols) {
+      return prepareFrame(source,response != null?new Vec[]{response}:null,ignored_cols,toEnum,dropConstantCols,dropNACols);
+    }
+    public static Frame prepareFrame(Frame source, Vec [] response, int[] ignored_cols, boolean toEnum, boolean dropConstantCols, boolean dropNACols) {
       Frame fr = new Frame(Key.makeSystem(Key.make().toString()), source._names.clone(), source.vecs().clone());
       if (ignored_cols != null) fr.remove(ignored_cols);
       final Vec[] vecs =  fr.vecs();
@@ -159,21 +178,13 @@ public static Frame prepareFrame(Frame source, Vec response, int[] ignored_cols,
 
       // put response to the end (if not already)
       if (response != null) {
-        for (int i = 0; i < vecs.length - 1; ++i) {
-          if (vecs[i] == response) {
-            final String n = fr._names[i];
-            if (toEnum && !vecs[i].isEnum()) fr.add(n, fr.remove(i).toEnum()); //convert int classes to enums
-            else fr.add(n, fr.remove(i));
-            break;
-          }
-        }
-        // special case for when response was at the end already
-        if (toEnum && !response.isEnum() && vecs[vecs.length - 1] == response) {
-          final String n = fr._names[vecs.length - 1];
-          fr.add(n, fr.remove(vecs.length - 1).toEnum());
+        for(Vec v:response){
+          int id = fr.find(v);
+          final String n = fr._names[id];
+          if (toEnum && !vecs[id].isEnum()) fr.add(n, fr.remove(id).toEnum()); //convert int classes to enums
+          else fr.add(n, fr.remove(id));
         }
       }
-
       ArrayList<Integer> constantOrNAs = new ArrayList<Integer>();
       {
         ArrayList<Integer> constantCols = new ArrayList<Integer>();