Skip to content

Commit

Permalink
Fix POJO generator
Browse files Browse the repository at this point in the history
  - missing semicolon in POJO generator
  - make file name and class name consistent
  - do not generate NAMES field if it is too long
    - Right now each string in NAMES field occupies 4bytes in the static initializer of class,
so the limit is ~ 65536/(2*2*4) since we need to load all strings from NAMES array and ColInfo
in static initializer. And static initializer influences the size of
resulting class.
  - do not generate ColInfo if it is not necessary.
  • Loading branch information
mmalohlava committed May 29, 2015
1 parent 3663a6b commit 87a5201
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 10 deletions.
4 changes: 2 additions & 2 deletions src/main/java/hex/gbm/GBM.java
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ private GBMModel(GBMModel prior, Key[][] treeKeys, double[] errs, ConfusionMatri
if(family == Family.bernoulli) {
bodyCtxSB.i().p("// Compute Probabilities for Bernoulli 0-1 classifier").nl();
bodyCtxSB.i().p("double fx = preds[1] + "+initialPrediction+";").nl();
bodyCtxSB.i().p("preds[2] = 1.0f/(float)(1.0f+Math.exp(-fx))").nl();
bodyCtxSB.i().p("preds[1] = 1.0f-preds[2]").nl();
bodyCtxSB.i().p("preds[2] = 1.0f/(float)(1.0f+Math.exp(-fx));").nl();
bodyCtxSB.i().p("preds[1] = 1.0f-preds[2];").nl();
}
else if (isClassifier()) {
bodyCtxSB.i().p("// Compute Probabilities for classifier (scale via http://www.hongliangjie.com/2011/01/07/logsum/)").nl();
Expand Down
10 changes: 8 additions & 2 deletions src/main/java/water/Model.java
Original file line number Diff line number Diff line change
Expand Up @@ -656,7 +656,13 @@ protected SB toJavaSuper( SB sb ) {

return sb;
}
private SB toJavaNAMES( SB sb ) { return JCodeGen.toStaticVar(sb, "NAMES", _names, "Names of columns used by model."); }
private SB toJavaNAMES( SB sb ) {
//
int limit = ((1<<16) /* Max size of class */ - 4 * 500 /* Free space for loading any static stuff */ ) / (4*2*2); // Static initialized needs 4 instructions to load String from constant pool + load ColInfo
return _names.length < limit ?
JCodeGen.toStaticVar(sb, "NAMES", _names, "Names of columns used by model.") :
JCodeGen.toStaticVar(sb, "NAMES", JCodeGen.EMPTY_SA, "Names of columns used by model. WARNING: It is too large to be generated!");
}
protected SB toJavaNCLASSES( SB sb ) { return isClassifier() ? JCodeGen.toStaticVar(sb, "NCLASSES", nclasses(), "Number of output classes included in training data response column.") : sb; }
private SB toJavaDOMAINS( SB sb, SB fileContextSB ) {
sb.nl();
Expand All @@ -667,7 +673,7 @@ private SB toJavaDOMAINS( SB sb, SB fileContextSB ) {
String[] dom = _domains[i];
String colInfoClazz = "ColInfo_"+i;
sb.i(1).p("/* ").p(_names[i]).p(" */ ");
sb.p(colInfoClazz).p(".VALUES");
if (dom != null) sb.p(colInfoClazz).p(".VALUES"); else sb.p("null");
if (i!=_domains.length-1) sb.p(',');
sb.nl();
fileContextSB.i().p("// The class representing column ").p(_names[i]).nl();
Expand Down
15 changes: 9 additions & 6 deletions src/main/java/water/api/SaveModel.java
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package water.api;

import java.io.*;

import static water.util.FSUtils.isHdfs;
import static water.util.FSUtils.isS3N;

import java.io.File;
import java.io.IOException;

import hex.glm.GLMModel;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

Expand All @@ -16,6 +18,7 @@
import water.serial.Model2FileBinarySerializer;
import water.serial.Model2HDFSBinarySerializer;
import water.util.FSUtils;
import water.util.JCodeGen;

public class SaveModel extends Func {
static final int API_WEAVER = 1;
Expand Down Expand Up @@ -48,7 +51,7 @@ private void saveToLocalFS() {
// Create folder
parentDir.mkdirs();
// Save parent model
new Model2FileBinarySerializer().save(model, new File(parentDir, model._key.toString()));
new Model2FileBinarySerializer().save(model, new File(parentDir, JCodeGen.toJavaId(model._key.toString())));
// Write to model_names
File model_names = new File(parentDir, "model_names");
FileOutputStream is = new FileOutputStream(model_names);
Expand All @@ -61,8 +64,8 @@ private void saveToLocalFS() {
Model[] models = getCrossValModels(model);
System.out.println(models);
for (Model m : models) {
new Model2FileBinarySerializer().save(m, new File(parentDir, m._key.toString()));
br.write(m._key.toString());
new Model2FileBinarySerializer().save(m, new File(parentDir, JCodeGen.toJavaId(m._key.toString())));
br.write(JCodeGen.toJavaId(m._key.toString()));
br.newLine();
}
}
Expand All @@ -80,7 +83,7 @@ private void saveToHdfs() {
if (force && fs.exists(parentDir)) fs.delete(parentDir);
fs.mkdirs(parentDir);
// Save parent model
new Model2HDFSBinarySerializer(fs, force).save(model, new Path(parentDir, model._key.toString()));
new Model2HDFSBinarySerializer(fs, force).save(model, new Path(parentDir, JCodeGen.toJavaId(model._key.toString())));
// Save parent model key to model_names file
Path model_names = new Path(parentDir, "model_names");
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(model_names,true)));
Expand All @@ -89,8 +92,8 @@ private void saveToHdfs() {
if (save_cv) {
Model[] models = getCrossValModels(model);
for (Model m : models ) {
new Model2HDFSBinarySerializer(fs, force).save(m, new Path(parentDir, m._key.toString()));
br.write(m._key.toString());
new Model2HDFSBinarySerializer(fs, force).save(m, new Path(parentDir, JCodeGen.toJavaId(m._key.toString())));
br.write(JCodeGen.toJavaId(m._key.toString()));
br.newLine();
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/water/util/JCodeGen.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

public class JCodeGen {

public static final String[] EMPTY_SA = new String[] {} ;

/** Generates data sample as a dedicated class with static <code>double[][]</code> member. */
public static SB toClass(SB sb, String classSig, String varname, Frame f, int nrows, String comment) {
sb.p(classSig).p(" {").nl().ii(1);
Expand Down

0 comments on commit 87a5201

Please sign in to comment.