univatiate distr based predictor

KCSekhar-DE · Jan 22, 2019 · 1670138 · 1670138
1 parent 9170fbc
commit 1670138
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 7 deletions.
diff --git a/spark/src/main/scala/org/beymani/spark/dist/StatsBasedOutlierPredictor.scala b/spark/src/main/scala/org/beymani/spark/dist/StatsBasedOutlierPredictor.scala
@@ -32,8 +32,9 @@ import org.chombo.spark.common.SeasonalUtility
 import org.beymani.predictor.EstimatedProbabilityBasedPredictor
 import org.beymani.predictor.EsimatedAttrtibuteProbabilityBasedPredictor
 import org.apache.spark.Accumulator
+import org.chombo.spark.common.GeneralUtility
 
-object StatsBasedOutlierPredictor extends JobConfiguration with SeasonalUtility {
+object StatsBasedOutlierPredictor extends JobConfiguration with SeasonalUtility with GeneralUtility{
    private val predStrategyZscore = "zscore";
    private val predStrategyRobustZscore = "robustZscore";
    private val predStrategyEstProb = "estimatedProbablity";
@@ -55,7 +56,12 @@ object StatsBasedOutlierPredictor extends JobConfiguration with SeasonalUtility
 	   //configuration params
 	   val fieldDelimIn = appConfig.getString("field.delim.in")
 	   val fieldDelimOut = appConfig.getString("field.delim.out")
-	   val predictorStrategy = getStringParamOrElse(appConfig, "predictor.strategy", predStrategyZscore)
+
+	   val predictorStrategy = getMandatoryStringParam(appConfig, "predictor.strategy", "missing prediction strategy")
+	   val predictorStrategies = Array[String]("zscore", "robustZscore", "estimatedProbablity", 
+	       "estimatedAttributeProbablity", "extremeValueProbablity")
+	   assertStringMember(predictorStrategy, predictorStrategies, "invalid prediction strategy " + predictorStrategy)
+
 	   val appAlgoConfig = appConfig.getConfig(predictorStrategy)
 	   val algoConfig = getConfig(predictorStrategy, appConfig, appAlgoConfig)
 	   val scoreThreshold:java.lang.Double = getMandatoryDoubleParam(appConfig, "score.threshold", "missing score threshold")
@@ -353,10 +359,12 @@ object StatsBasedOutlierPredictor extends JobConfiguration with SeasonalUtility
 	       configParams.put("schema.filePath", schemaFilePath)
 	     }
 	     case `predStrategyEstAttrProb` => {
+	       val attWeightList = getMandatoryDoubleListParam(appAlgoConfig, "attr.weights", "missing attribute weights")
+	       val attrWeights = BasicUtils.fromListToDoubleArray(attWeightList)
+	       configParams.put("attr.weights", attrWeights)
 	       val distrFilePath = getMandatoryStringParam(appAlgoConfig, "distr.file.path", "missing distr file path")
 	       configParams.put("distr.filePath", distrFilePath)
-	       val schemaFilePath = getMandatoryStringParam(appAlgoConfig, "schema.file.path", "missing schema file path")
-	       configParams.put("schema.filePath", schemaFilePath)
+	       configParams.put("schema.filePath", null)
 	     }
 	   }
 

diff --git a/src/main/java/org/beymani/predictor/DistributionBasedPredictor.java b/src/main/java/org/beymani/predictor/DistributionBasedPredictor.java
@@ -151,6 +151,7 @@ public DistributionBasedPredictor(Map<String, Object> config, String idOrdinalsP
 	 */
 
 	/**
+	 * uni variate distribution
 	 * @param config
 	 * @param idOrdinalsParam
 	 * @param distrFilePathParam
@@ -187,15 +188,23 @@ public DistributionBasedPredictor(Map<String, Object> config, String idOrdinalsP
 			this.keyedHist.put(compKey, keyedHist.get(key));
 		}
 
-       	String schemFilePath = ConfigUtility.getString(config, schemaFilePathParam);
-    	schema = BasicUtils.getRichAttributeSchema(schemFilePath);
+		if (null != config.get(schemaFilePathParam)) {
+			String schemFilePath = ConfigUtility.getString(config, schemaFilePathParam);
+       		schema = BasicUtils.getRichAttributeSchema(schemFilePath);
+		}
     	scoreThreshold = ConfigUtility.getDouble(config, scoreThresholdParam);
 	}
 
 	/**
 	 * for multi variate distribution
 	 * @param config
-	 * @param distrFilePath
+	 * @param idOrdinalsParam
+	 * @param distrFilePathParam
+	 * @param hdfsFileParam
+	 * @param schemaFilePathParam
+	 * @param seasonalParam
+	 * @param fieldDelimParam
+	 * @param scoreThresholdParam
 	 * @throws IOException
 	 */
 	public DistributionBasedPredictor(Map<String, Object> config, String idOrdinalsParam, String distrFilePathParam,