Adapted models to deal with different sets of categoricals in training and tst set.

Model now has method adapt(ValueArray) and adapt(String[]) which produces model adapted for different dataset. Default implementations provided but the Model.class which simply maps the columns and categoricals to their matching pair. Categoricals with no matching value (seen only in testing data, not in training) are mapped to NaN.

Also, I removed mapping argument from all score methods.

Author: tomasnykodym

src/main/java/hex/DGLM.java

@@ -682,8 +682,8 @@ protected double score0( double[] data ) {
           p += _beta[idx]*d;
         } else {
           int d = (int)data[i]; // Enum value
-          idx += d;             // Which expanded column to use
-          if( idx < _colCatMap[i+1] )
+          // d can be -1 if we got enum values not seen in training
+          if(d > 0 && (idx += d) < _colCatMap[i+1])
             p += _beta[idx]/* *1.0 */;
           else             // Enum out of range?
             p = Double.NaN;// Can use a zero, or a NaN
@@ -697,12 +697,12 @@ protected double score0( double[] data ) {
     }
 
     /** Single row scoring, on a compatible ValueArray (when pushed throw the mapping) */
-    protected double score0( ValueArray data, int row, int[] mapping ) {
+    protected double score0( ValueArray data, int row) {
       throw H2O.unimpl();
     }
 
     /** Bulk scoring API, on a compatible ValueArray (when pushed throw the mapping) */
-    protected double score0( ValueArray data, AutoBuffer ab, int row_in_chunk, int[] mapping ) {
+    protected double score0( ValueArray data, AutoBuffer ab, int row_in_chunk) {
       throw H2O.unimpl();
     }
   }

src/main/java/hex/KMeansModel.java

@@ -88,12 +88,12 @@ protected double score0(double[] data) {
   }
 
   /** Single row scoring, on a compatible ValueArray (when pushed throw the mapping) */
-  protected double score0(ValueArray data, int row, int[] mapping) {
+  protected double score0(ValueArray data, int row) {
     throw H2O.unimpl();
   }
 
   /** Bulk scoring API, on a compatible ValueArray (when pushed throw the mapping) */
-  protected double score0(ValueArray data, AutoBuffer ab, int row_in_chunk, int[] mapping) {
+  protected double score0(ValueArray data, AutoBuffer ab, int row_in_chunk) {
     throw H2O.unimpl();
   }
 

src/main/java/hex/rf/RFModel.java

@@ -203,10 +203,10 @@ protected double score0(double[] data) {
   }
 
   /** Single row scoring, on a compatible ValueArray (when pushed throw the mapping) */
-  protected double score0( ValueArray data, int row, int[] mapping ) { throw H2O.unimpl(); }
+  protected double score0( ValueArray data, int row) { throw H2O.unimpl(); }
 
   /** Bulk scoring API, on a compatible ValueArray (when pushed throw the mapping) */
-  protected double score0(ValueArray data, AutoBuffer ab, int row_in_chunk, int[] mapping) { throw H2O.unimpl(); }
+  protected double score0(ValueArray data, AutoBuffer ab, int row_in_chunk) { throw H2O.unimpl(); }
 
   @Override public JsonObject toJson() {
     JsonObject res = new JsonObject();

src/main/java/water/InternalInterface.java

@@ -33,14 +33,18 @@ public double scoreKey( Object modelKey, String [] colNames, double[] row ) {
     }
   }
 
+  public Model adaptModel(Object model, String [] colNames){
+    return ((Model)model).adapt(colNames);
+  }
   // Call to map the columns and score
-  public double scoreModel( Object model, String [] colNames, double[] row ) {
-    Model M = (Model)model;
-    int[] map = M.columnMapping( colNames);
-    if( !Model.isCompatible(map) )
-      throw new IllegalArgumentException("This model uses different columns than those provided");
-    return M.score(row,map);
+  public double scoreModel( Object model,double[] row ) {
+    return ((Model)model).score(row);
   }
 
   public JsonObject cloudStatus( ) { return new Cloud().serve().toJson(); }
+
+  @Override public double scoreModel(Object model, String[] colNames, double[] row) {
+    return adaptModel(model, colNames).score(row);
+  }
+
 }

src/main/java/water/Model.java

@@ -2,10 +2,11 @@
 
 import java.util.Arrays;
 
-import com.google.gson.JsonObject;
-
+import water.ValueArray.Column;
 import water.api.Constants;
 
+import com.google.gson.JsonObject;
+
 /**
  * A Model models reality (hopefully).
  * A model can be used to 'score' a row, or a collection of rows on any
@@ -168,36 +169,116 @@ public final boolean isCompatible( ValueArray data ) {
     //return isCompatible(data.colNames());
   }
 
-  /** Single row scoring.  Data can be in any order.  No checking on a sane
-   *  mapping. */
-  public final double score( double[] data, int[] mapping ) {
-    assert isCompatible(mapping);
-    if( identityMap(mapping) ) { // Shortcut for well-behaved data
-      assert data.length == _va._cols.length;
-      return score0(data);
+  private static class ModelDataAdaptor extends Model {
+    final Model M;
+    final int       _yCol;
+    final int  []   _xCols;
+    final int  [][] _catMap;
+    final double [] _row;
+
+    public ModelDataAdaptor(Model M, int yCol, int [] cols, int [][] catMap){
+      this.M = M;
+      _row = MemoryManager.malloc8d(cols.length);
+      _xCols = cols;
+      _catMap = catMap;
+      _yCol = yCol;
+    }
+    private final double translateCat(int col, int val){
+      int res = _catMap[col][val];
+      return res == -1?Double.NaN:res;
+    }
+    private final double translateCat(int col, double val){
+      if(Double.isNaN(val))return val;
+      assert val == (int)val;
+      return translateCat(col, (int)val);
+    }
+
+    @Override public final double score(double[] data) {
+      int j = 0;
+      for(int i = 0; i < _xCols.length; ++i)
+        _row[j++] = (_catMap == null || _catMap[i] == null)?data[_xCols[i]]:translateCat(i, data[_xCols[i]]);
+      return M.score0(_row);
+    }
+    @Override protected double score0(ValueArray data, int row) {
+      int j = 0;
+      for(int c:_xCols)
+        _row[j++] = (_catMap == null || _catMap[c] == null)
+          ?data.datad(row, c)
+          :translateCat(c,(int)data.data(row, c));
+      return M.score0(_row);
+    }
+    @Override protected double score0(ValueArray data, AutoBuffer ab, int row) {
+      int j = 0;
+      for(int c:_xCols)
+        _row[j++] = (_catMap == null || _catMap[c] == null)
+          ?data.datad(ab,row, c)
+              :translateCat(c,(int)data.data(ab,row, c));
+      return M.score0(_row);
     }
-    // Build a mapped row and score it.  Explodes if mapping is busted.
-    double[] d = new double[_va._cols.length];
-    for( int i=0; i<_va._cols.length-1; i++ )
-      d[i] = data[mapping[i]];
-    return score0(d);
+    // always should call directly M.score0...
+    @Override protected final double score0(double[] data) {
+      throw new RuntimeException("should NEVER be called!");
+    }
+    @Override public JsonObject toJson() {return M.toJson();}
+    // keep only one adaptor layer! (just in case there would be multiple adapt calls...)
+    @Override public final Model adapt(ValueArray ary){return M.adapt(ary);}
+    @Override public final Model adapt(String [] cols){return M.adapt(cols);}
+  }
+
+  /**
+   * Adapt model for the given dataset.
+   * Default behavior is to map columns and categoricals to their original indexes.
+   *
+   * @param ary - tst dataset
+   * @return Model - model adapted to be applied on the given data
+   */
+  public Model adapt(ValueArray ary){
+    boolean id = true;
+    final int  [] colMap = columnMapping(ary.colNames());
+    if(!isCompatible(colMap))throw new IllegalArgumentException("This model uses different columns than those provided");
+    final int[][] catMap =  new int[colMap.length][];
+    for(int i = 0; i < colMap.length-1; ++i){
+      Column c = ary._cols[colMap[i]];
+      if(c.isEnum() && !Arrays.deepEquals(_va._cols[i]._domain, c._domain)){
+        id = false;
+        catMap[i] = new int[c._domain.length];
+        for(int j = 0; j < c._domain.length; ++j)
+          catMap[i][j] = find(c._domain[j],_va._cols[i]._domain);
+      }
+    }
+    return (id&&identityMap(colMap))?this:new ModelDataAdaptor(this,colMap[colMap.length-1],Arrays.copyOf(colMap,colMap.length-1),catMap);
+  }
+  /**
+   * Adapt model for given columns.
+   * Only permutes the columns by the column names (factor levels MUST match the training dataset).
+   * @param colNames
+   * @return
+   */
+  public Model adapt(String [] colNames){
+    final int [] colMap = columnMapping(colNames);
+    if(!isCompatible(colMap))throw new IllegalArgumentException("This model uses different columns than those provided");
+    if(identityMap(colMap))return this;
+    return new ModelDataAdaptor(this, colMap[colMap.length-1],Arrays.copyOf(colMap,colMap.length-1), null);
+  }
+  public double score(double [] data){
+    return score0(data);
   }
 
   // Subclasses implement the scoring logic.  They can assume all datasets are
   // compatible already
+  protected abstract double score0(double [] data);
 
-  /** Single row scoring, on properly ordered data */
-  protected abstract double score0( double[] data );
 
   /** Single row scoring, on a compatible ValueArray (when pushed throw the mapping) */
-  protected abstract double score0( ValueArray data, int row, int[] mapping );
+  protected abstract double score0( ValueArray data, int row);
 
   /** Bulk scoring API, on a compatible ValueArray (when pushed throw the mapping) */
-  protected abstract double score0( ValueArray data, AutoBuffer ab, int row_in_chunk, int[] mapping );
+  protected abstract double score0( ValueArray data, AutoBuffer ab, int row_in_chunk);
 
   public abstract JsonObject toJson();
 
   public void fromJson(JsonObject json) {
     // TODO
   }
+
 }

src/main/java/water/api/Score.java

@@ -87,7 +87,7 @@ public static Score create( Properties parms ) {
     res.add(ROWS, rows);
 
     // Score the row on the model.  May destroy 'd'.
-    double response = M.score(d,null);
+    double response = M.score(d);
     res.addProperty(CLASS, response);
 
     // Display HTML setup

src/test/java/water/parser/RReaderTest.java

@@ -22,17 +22,15 @@ public void testIrisModel() throws Exception {
     Key key = Key.make("irisModel");
     RReader.run(key, new FileInputStream(file));
     RFModel model = UKV.get(key);
-    int[] map = model.columnMapping(iris.colNames());
-    Assert.assertTrue(Model.isCompatible(map));
+    Model m = model.adapt(iris.colNames());
 
     // Can I score on the model now?
-    double[] row = new double[map.length];
+    double[] row = new double[iris._cols.length];
     for( int i=0; i<iris._numrows; i++ ) {
-      for( int j=0; j<map.length-1; j++ )
+      for( int j=0; j<iris._cols.length-1; j++ )
         row[j] = iris.datad(i,j);
-      assertEquals(iris.datad(i,map.length-1),model.score(row,map),0.0001);
+      assertEquals(iris.datad(i,iris._cols.length-1),m.score(row),0.0001);
     }
-
     model.deleteKeys();
     UKV.remove(key);
     UKV.remove(irisk);
@@ -54,10 +52,11 @@ public void testProstateModel() throws Exception {
     // Can I score on the model now?
     double[] row = new double[map.length];
     int errs = 0;
+    Model M = model.adapt(pro);
     for( int i=0; i<pro._numrows; i++ ) {
       for( int j=0; j<map.length; j++ )
         row[j] = pro.datad(i,j);
-      double score = model.score(row,map);
+      double score = M.score(row);
       if( Math.abs(pro.datad(i,classCol) - score) > 0.0001 ) errs++;
     }
     assertEquals(100,errs);
@@ -84,11 +83,12 @@ public void testCovtypeModel() throws Exception {
     // Can I score on the model now?
     long start = System.currentTimeMillis();
     double[] row = new double[map.length];
+    Model M = model.adapt(pro);
     int errs = 0;
     for( int i=0; i<pro._numrows; i++ ) {
       for( int j=0; j<map.length; j++ )
         row[j] = pro.datad(i,j);
-      double score = model.score(row,map);
+      double score = M.score(row);
       System.out.println(" "+i+" "+score+" "+(pro.datad(i,classCol)));
       if( Math.abs(pro.datad(i,classCol) - score) > 0.0001 ) errs++;
     }