trim fields

KCSekhar-DE · Oct 30, 2018 · e5d770b · e5d770b
1 parent 168d8f5
commit e5d770b
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/resource/cyd.sh b/resource/cyd.sh
@@ -34,6 +34,13 @@ case "$1" in
 	--conf spark.ui.killEnabled=true --master $MASTER $CHOMBO_JAR_NAME  $INPUT $OUTPUT cyd.conf
 ;;
 
+"crAucInput")
+	echo "copying and consolidating tem aggregation output file"
+	cat $PROJECT_HOME/bin/beymani/output/teg/part-00000 > $PROJECT_HOME/bin/beymani/input/auc/cusage.txt
+	cat $PROJECT_HOME/bin/beymani/output/teg/part-00001 >> $PROJECT_HOME/bin/beymani/input/auc/cusage.txt
+	ls -l $PROJECT_HOME/bin/beymani/input/auc
+;;
+
 "autoCor")
 	echo "running AutoCorrelation Spark job"
 	CLASS_NAME=org.chombo.spark.explore.AutoCorrelation

diff --git a/spark/src/main/scala/org/beymani/spark/dist/StatsBasedOutlierPredictor.scala b/spark/src/main/scala/org/beymani/spark/dist/StatsBasedOutlierPredictor.scala
@@ -163,7 +163,7 @@ object StatsBasedOutlierPredictor extends JobConfiguration with SeasonalUtility
 	  keyLen += (if (seasonalAnalysis && partBySeasonCycle) 1 else 0)
 
 	 var taggedData = data.map(line => {
-		   val items = line.split(fieldDelimIn, -1)
+		   val items = BasicUtils.getTrimmedFields(line, fieldDelimIn)
 		   val key = Record(keyLen)
 		   //partioning fields
 		   keyFieldOrdinals match {