Skip to content

Commit

Permalink
config param key prefix
Browse files Browse the repository at this point in the history
  • Loading branch information
Pranab Ghosh committed Mar 28, 2016
1 parent 19c2597 commit 9035d8b
Show file tree
Hide file tree
Showing 6 changed files with 29 additions and 28 deletions.
8 changes: 4 additions & 4 deletions src/main/java/org/beymani/dist/DistributionSorter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,13 @@ protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelimRegex = conf.get("field.delim.regex", "\\[\\]");

String filePath = conf.get("histogram.schema.file.path");
String filePath = conf.get("dis.histogram.schema.file.path");
FileSystem dfs = FileSystem.get(conf);
Path src = new Path(filePath);
FSDataInputStream fs = dfs.open(src);
ObjectMapper mapper = new ObjectMapper();
schema = mapper.readValue(fs, RichAttributeSchema.class);
itemDelim = conf.get("item.delim", ",");
itemDelim = conf.get("dis.item.delim", ",");

List<Byte> dataTypes = new ArrayList<Byte>();
for (RichAttribute field : schema.getFields()) {
Expand Down Expand Up @@ -154,9 +154,9 @@ public static class SorterReducer extends Reducer<Tuple, Text, NullWritable, Tex

protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
maxItemCount = conf.getInt("max.item.count", -1);
maxItemCount = conf.getInt("dis.max.item.count", -1);
itemCount = 0;
itemDelim = conf.get("item.delim", ",");
itemDelim = conf.get("dis.item.delim", ",");
filtered = maxItemCount > 0;
}

Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/beymani/dist/EntropyBasedDistributionSorter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,14 +97,14 @@ protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelimRegex = conf.get("field.delim.regex", "\\[\\]");

String filePath = conf.get("histogram.schema.file.path");
String filePath = conf.get("ebd.histogram.schema.file.path");
FileSystem dfs = FileSystem.get(conf);
Path src = new Path(filePath);
FSDataInputStream fs = dfs.open(src);
ObjectMapper mapper = new ObjectMapper();
schema = mapper.readValue(fs, RichAttributeSchema.class);
itemDelim = conf.get("item.delim", ",");
totalItemCount = conf.getInt("total.Item.count", -1);
itemDelim = conf.get("ebd.item.delim", ",");
totalItemCount = conf.getInt("ebd.total.Item.count", -1);
if (totalItemCount == -1) {
throw new IllegalStateException("max item count should be provided");
}
Expand Down Expand Up @@ -147,11 +147,11 @@ public static class SorterReducer extends Reducer<Tuple, Tuple, NullWritable, Te
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelim = conf.get("field.delim", "[]");
maxItemCount = conf.getInt("max.item.count", -1);
maxItemCount = conf.getInt("ebd.max.item.count", -1);
itemCount = 0;
itemDelim = conf.get("item.delim", ",");
itemDelim = conf.get("ebd.item.delim", ",");
filtered = maxItemCount > 0;
outputEntropy = conf.getBoolean("output.entropy", false);
outputEntropy = conf.getBoolean("ebd.output.entropy", false);
}

protected void reduce(Tuple key, Iterable<Text> values, Context context)
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/beymani/dist/MultiVariateDistribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelimRegex = conf.get("field.delim.regex", "\\[\\]");

String filePath = conf.get("histogram.schema.file.path");
String filePath = conf.get("mvd.histogram.schema.file.path");
FileSystem dfs = FileSystem.get(conf);
Path src = new Path(filePath);
FSDataInputStream fs = dfs.open(src);
Expand All @@ -103,7 +103,7 @@ protected void setup(Context context) throws IOException, InterruptedException {
numFields = schema.getFields().size();
partitionField = schema.getPartitionField();
idField = schema.getIdField();
fieldOrdinals = Utility.intArrayFromString(conf.get("hist.field.ordinals"));
fieldOrdinals = Utility.intArrayFromString(conf.get("mvd.hist.field.ordinals"));
}

@Override
Expand Down Expand Up @@ -179,8 +179,8 @@ public static class HistogramReducer extends Reducer<Tuple, Text, NullWritable,
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelim = conf.get("field.delim", "[]");
itemDelim = conf.get("item.delim", ",");
outputCount = conf.getBoolean("output.count", false);
itemDelim = conf.get("mvd.item.delim", ",");
outputCount = conf.getBoolean("mvd.output.count", false);
}

protected void reduce(Tuple key, Iterable<Text> values, Context context)
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/beymani/dist/MultigramMultiVariateDistribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelimRegex = conf.get("field.delim.regex", "\\[\\]");

String filePath = conf.get("histogram.schema.file.path");
String filePath = conf.get("mmvd.histogram.schema.file.path");
FileSystem dfs = FileSystem.get(conf);
Path src = new Path(filePath);
FSDataInputStream fs = dfs.open(src);
Expand All @@ -118,13 +118,13 @@ protected void setup(Context context) throws IOException, InterruptedException {
numFields = schema.getFields().size();
partitionField = schema.getPartitionField();
idField = schema.getIdField();
fieldOrdinals = Utility.intArrayFromString(conf.get("hist.field.ordinals"));
fieldOrdinals = Utility.intArrayFromString(conf.get("mmvd.hist.field.ordinals"));
if (null != fieldOrdinals) {
fieldCount = fieldOrdinals.length;
} else {
fieldCount = schema.getAttributeCount(true, true);
}
sequenceLength = conf.getInt("sequence.length", 3);
sequenceLength = conf.getInt("mmvd.sequence.length", 3);
}

@Override
Expand Down
19 changes: 10 additions & 9 deletions src/main/java/org/beymani/dist/StatsBasedOutlierPredictor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public int run(String[] args) throws Exception {
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

Utility.setConfiguration(job.getConfiguration(), "chombo");
Utility.setConfiguration(job.getConfiguration(), "beymani");
job.setMapperClass(StatsBasedOutlierPredictor.PredictorMapper.class);

job.setOutputKeyClass(NullWritable.class);
Expand Down Expand Up @@ -87,19 +87,20 @@ public static class PredictorMapper extends Mapper<LongWritable, Text, NullWrita
protected void setup(Context context) throws IOException, InterruptedException {
Configuration config = context.getConfiguration();
fieldDelim = config.get("field.delim.out", ",");
predictorStartegy = config.get("predictor.startegy", PRED_STRATEGY_ZSCORE);
predictorStartegy = config.get("sbop.predictor.startegy", PRED_STRATEGY_ZSCORE);

if (predictorStartegy.equals(PRED_STRATEGY_ZSCORE)) {
predictor = new ZscorePredictor(config, "id.field.ordinals", "attr.list", "stats.file.path", "field.delim.regex",
"attr.weight", "score.threshold");
predictor = new ZscorePredictor(config, "sbop.id.field.ordinals", "sbop.attr.list", "sbop.stats.file.path",
"field.delim.regex", "sbop.attr.weight", "sbop.score.threshold");
} else if (predictorStartegy.equals(PRED_STRATEGY_ROBUST_ZSCORE)) {
predictor = new RobustZscorePredictor(config, "id.field.ordinals", "attr.list", "med.stats.file.path", "mad.stats.file.path",
"field.delim.regex", "attr.weight", "score.threshold");
predictor = new RobustZscorePredictor(config, "sbop.id.field.ordinals", "sbop.attr.list",
"sbop.med.stats.file.path", "sbop.mad.stats.file.path", "field.delim.regex",
"sbop.attr.weight", "sbop.score.threshold");
} else if (predictorStartegy.equals(PRED_STRATEGY_EST_PROB)) {
predictor = new EstimatedProbabilityBasedPredictor(config, "distr.file.path", "score.threshold" );
predictor = new EstimatedProbabilityBasedPredictor(config, "sbop.distr.file.path", "sbop.score.threshold" );
} else if (predictorStartegy.equals(PRED_STRATEGY_EST_ATTR_PROB)) {
predictor = new EsimatedAttrtibuteProbabilityBasedPredictor(config, "distr.file.path", "attr.weight", "score.threshold",
"field.delim.regex");
predictor = new EsimatedAttrtibuteProbabilityBasedPredictor(config, "sbop.distr.file.path", "sbop.attr.weight",
"sbop.score.threshold", "field.delim.regex");
} else {
throw new IllegalArgumentException("ivalid predictor strategy");
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/beymani/predictor/DistributionBasedPredictor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ public DistributionBasedPredictor(Configuration config, String distrFilePath) th
distrModel.put(items[0], count);
}

schema = Utility.getRichAttributeSchema(config, "distr.schema.file.path");
scoreThreshold = Double.parseDouble(config.get("score.threshold"));
schema = Utility.getRichAttributeSchema(config, "dbp.distr.schema.file.path");
scoreThreshold = Double.parseDouble(config.get("dbp.score.threshold"));
}

/**
Expand Down

0 comments on commit 9035d8b

Please sign in to comment.