updated pmml generation and evaluation

lobesoul · Oct 8, 2016 · bdcd6c6 · bdcd6c6
1 parent 43d99f8
commit bdcd6c6
Show file tree

Hide file tree

Showing 10 changed files with 472 additions and 39 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -75,10 +75,10 @@ ENV \
  JMETER_VERSION=3.0 \
  CODAHALE_METRICS_VERSION=3.1.2 \
  GUAVA_VERSION=14.0.1 \
- JPMML_SPARKML_VERSION=1.0.4 \
- PMML_MODEL_METRO_VERSION=1.2.15 \
- PMML_MODEL_VERSION=1.2.15 \
- PMML_EVALUATOR_VERSION=1.2.14 \
+ JPMML_SPARKML_VERSION=1.1.3 \
+ PMML_MODEL_VERSION=1.3.3 \
+ PMML_EVALUATOR_VERSION=1.3.2 \
+ PMML_METRO_VERSION=1.3.3 \
  SPRING_PROFILES_ACTIVE=local
 
 RUN \

diff --git a/Dockerfile.kube b/Dockerfile.kube
@@ -75,10 +75,10 @@ ENV \
  JMETER_VERSION=3.0 \
  CODAHALE_METRICS_VERSION=3.1.2 \
  GUAVA_VERSION=14.0.1 \
- JPMML_SPARKML_VERSION=1.0.4 \
- PMML_MODEL_METRO_VERSION=1.2.15 \
- PMML_MODEL_VERSION=1.2.15 \
- PMML_EVALUATOR_VERSION=1.2.14 \
+ JPMML_SPARKML_VERSION=1.1.3 \
+ PMML_MODEL_VERSION=1.3.3 \
+ PMML_EVALUATOR_VERSION=1.3.2 \
+ PMML_METRO_VERSION=1.3.3 \
  SPRING_PROFILES_ACTIVE=local
 
 RUN \

diff --git a/config/bash/pipeline.bashrc b/config/bash/pipeline.bashrc
@@ -1,3 +1,11 @@
+######
+export SPARK_CSV_CONNECTOR_VERSION=1.5.0
+export JPMML_SPARKML_VERSION=1.1.3
+export PMML_MODEL_VERSION=1.3.3
+export PMML_EVALUATOR_VERSION=1.3.2
+export PMML_METRO_VERSION=1.3.3
+######
+
 # Dev Install Home (Tools)
 #export DEV_INSTALL_HOME=~
 
@@ -91,7 +99,7 @@ export SPARK_SUBMIT_JARS=$MYAPPS_HOME/pmml/spark/1.6.1/lib/jpmml-sparkml-package
 # --py-files
 #export SPARK_SUBMIT_PYFILES=$MYAPPS_HOME/pmml/spark/1.6.1/lib/jpmml.py
 
-export SPARK_SUBMIT_ARGS="--jars $SPARK_SUBMIT_JARS --repositories $SPARK_REPOSITORIES --packages $SPARK_SUBMIT_PACKAGES"
+export SPARK_SUBMIT_ARGS="--driver-class-path $SPARK_SUBMIT_JARS --jars $SPARK_SUBMIT_JARS --repositories $SPARK_REPOSITORIES --packages $SPARK_SUBMIT_PACKAGES"
 # --py-files $SPARK_SUBMIT_PYFILES"
 export PYTHONPATH=/root/pipeline/myapps/pmml/spark/1.6.1/lib/jpmml_sparkml-1.0rc0-py2.7.egg
 

diff --git a/myapps/jupyter/Spark/PMML/CensusPMML.ipynb b/myapps/jupyter/Spark/PMML/CensusPMML.ipynb
diff --git a/myapps/pmml/spark/2.0.1/pom.xml b/myapps/pmml/spark/2.0.1/pom.xml
@@ -173,12 +173,12 @@
     <dependency>
       <groupId>org.jpmml</groupId>
       <artifactId>pmml-model</artifactId>
-      <version>1.3.1</version>
+      <version>1.3.3</version>
     </dependency>
     <dependency>
       <groupId>org.jpmml</groupId>
       <artifactId>pmml-evaluator</artifactId>
-      <version>1.3.1</version>
+      <version>1.3.2</version>
     </dependency>
 <!--
     <dependency>
@@ -190,7 +190,7 @@
     <dependency>
       <groupId>org.jpmml</groupId>
       <artifactId>jpmml-sparkml</artifactId>
-      <version>1.1.1</version>
+      <version>1.1.3</version>
     </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>

diff --git a/myapps/pmml/spark/2.0.1/src/main/scala/com/advancedspark/pmml/spark/ml/EvaluatePMML.scala b/myapps/pmml/spark/2.0.1/src/main/scala/com/advancedspark/pmml/spark/ml/EvaluatePMML.scala
@@ -78,7 +78,7 @@ object EvaluatePMML {
 
     val results = modelEvaluator.evaluate(arguments)
     val targetField = modelEvaluator.getTargetFields().asScala(0)
-    val targetValue = results.get(targetField)
+    val targetValue = results.get(targetField.getName)
 
     System.out.println(s"**** Predicted value for '${targetField.getName}': ${targetValue} ****")
   }

diff --git a/myapps/pmml/spark/2.0.1/src/main/scala/com/advancedspark/pmml/spark/ml/PMMLSparkML.scala b/myapps/pmml/spark/2.0.1/src/main/scala/com/advancedspark/pmml/spark/ml/PMMLSparkML.scala
@@ -127,7 +127,6 @@ object PMMLSparkML {
 
     System.out.println("Input schema:");
     System.out.println("\t" + "Input fields: " + inputFields) 
-//    System.out.println("\t" + "Group fields: " + modelEvaluator.getGroupFields())
 
     System.out.println("Output schema:");
     System.out.println("\t" + "Target fields: " + modelEvaluator.getTargetFields())
@@ -154,20 +153,13 @@ object PMMLSparkML {
         //   2) missing value treatment, 
         //   3) invalid value treatment 
         //   4) type conversion
-//        yield (inputField -> modelEvaluator.prepare(inputField, inputs(inputField.getValue)))
         yield (inputField.getName -> inputField.prepare(inputs(inputField.getName.getValue)))
       ).toMap.asJava
 
     val results = modelEvaluator.evaluate(arguments)
     val targetField = modelEvaluator.getTargetFields().asScala(0)
-    val targetValue = results.get(targetField)
+    val targetValue = results.get(targetField.getName)
 
     System.out.println(s"**** Predicted value for '${targetField.getName}': ${targetValue} ****")
-
-    //if (targetValue instanceof Computable) {
-    //    Computable computable = (Computable)targetValue;
-
-    //    Object primitiveValue = computable.getResult();
-    //}
   }
 }
diff --git a/myapps/serving/prediction/build.sbt b/myapps/serving/prediction/build.sbt
@@ -21,7 +21,7 @@ val springCloudVersion = sys.env("SPRING_CLOUD_VERSION")
 val springCoreVersion = sys.env("SPRING_CORE_VERSION")
 val pmmlEvaluatorVersion = sys.env("PMML_EVALUATOR_VERSION")
 val pmmlModelVersion = sys.env("PMML_MODEL_VERSION")
-val pmmlModelMetroVersion = sys.env("PMML_MODEL_METRO_VERSION")
+val pmmlMetroVersion = sys.env("PMML_METRO_VERSION")
 val janinoVersion = sys.env("JANINO_VERSION")
 val codahaleMetricsVersion = sys.env("CODAHALE_METRICS_VERSION")
 
@@ -33,14 +33,13 @@ lazy val deps = Seq(
   "org.springframework.boot" % "spring-boot-starter-web"   % springBootVersion,
   "org.scalanlp"         %% "breeze"                       % breezeVersion % "provided",
   "com.netflix.dyno"     % "dyno-jedis"                    % dynoVersion,
-  "org.jpmml"            % "pmml-evaluator"                % pmmlEvaluatorVersion,
   "org.springframework.boot" % "spring-boot-starter-actuator" % springBootVersion,
   "org.springframework.cloud" % "spring-cloud-starter-spectator" % springCloudVersion,
   "org.springframework.cloud" % "spring-cloud-starter-eureka" % springCloudVersion,
   "org.springframework.cloud" % "spring-cloud-starter-hystrix" % springCloudVersion,
   "org.springframework.cloud" % "spring-cloud-starter-atlas" % springCloudVersion,
   "org.springframework.cloud" % "spring-cloud-starter-config" % springCloudVersion,
-  "org.jpmml" % "pmml-model-metro" % pmmlModelMetroVersion,
+  "org.jpmml" % "pmml-model-metro" % pmmlMetroVersion,
   "org.jpmml" % "pmml-model" % pmmlModelVersion,
   "org.jpmml" % "pmml-evaluator" % pmmlEvaluatorVersion 
 )
diff --git a/...ng/prediction/src/main/scala/com/advancedspark/serving/prediction/PredictionService.scala b/...ng/prediction/src/main/scala/com/advancedspark/serving/prediction/PredictionService.scala
@@ -242,14 +242,13 @@ class PredictionService {
     import java.io.FileInputStream
     import org.jpmml.model.MetroJAXBUtil
     import org.xml.sax.InputSource
-    import org.jpmml.evaluator.TreeModelEvaluator
     import org.jpmml.evaluator.ModelEvaluatorFactory
     import org.jpmml.evaluator.Evaluator
     import org.jpmml.model.ImportFilter
     import org.jpmml.model.JAXBUtil
 
     try {
-      val inputMap = JSON.parseFull(inputs).get.asInstanceOf[Map[String,Any]]
+      val inputsMap = JSON.parseFull(inputs).get.asInstanceOf[Map[String,Any]]
 
       val pmml: java.io.File = new java.io.File(s"data/${pmmlName}/${pmmlName}.pmml")
 
@@ -261,25 +260,36 @@ class PredictionService {
 
       val modelEvaluatorFactory = ModelEvaluatorFactory.newInstance()
 
-      val modelEvaluator: Evaluator = modelEvaluatorFactory.newModelManager(pmml2)
+      val modelEvaluator: Evaluator = modelEvaluatorFactory.newModelEvaluator(pmml2)
+      System.out.println("Mining function: " + modelEvaluator.getMiningFunction())
 
-      val activeFields = modelEvaluator.getActiveFields().asScala
+      val inputFields = modelEvaluator.getInputFields().asScala
+
+      System.out.println("Input schema:");
+      System.out.println("\t" + "Input fields: " + inputFields)
+//    System.out.println("\t" + "Group fields: " + modelEvaluator.getGroupFields())
+
+      System.out.println("Output schema:");
+      System.out.println("\t" + "Target fields: " + modelEvaluator.getTargetFields())
+      System.out.println("\t" + "Output fields: " + modelEvaluator.getOutputFields())
 
       val arguments =
-        ( for(activeField <- activeFields)
-        // The raw value is passed through:
-        //   1) outlier treatment,
-        //   2) missing value treatment,
-        //   3) invalid value treatment
-        //   4) type conversion
-          yield (activeField -> modelEvaluator.prepare(activeField, inputMap(activeField.getValue)))
+        ( for(inputField <- inputFields)
+          // The raw value is passed through:
+          //   1) outlier treatment,
+          //   2) missing value treatment,
+          //   3) invalid value treatment
+          //   4) type conversion
+          yield (inputField.getName -> inputField.prepare(inputsMap(inputField.getName.getValue)))
         ).toMap.asJava
 
+      System.out.println(arguments)
+
       val results = modelEvaluator.evaluate(arguments)
-      val targetName = modelEvaluator.getTargetField()
-      val targetValue = results.get(targetName)
+      val targetField = modelEvaluator.getTargetFields().asScala(0)
+      val targetValue = results.get(targetField.getName)
 
-      s"""{"results":[{'${targetName.getValue}': '${targetValue}'}]}"""
+      s"""{"results":[{'${targetField.getName}': '${targetValue}'}]}"""
     } catch {
        case e: Throwable => {
          System.out.println(e)