Allstate insurance Amount Prediction - Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Business Case : Predict insurance amount based on historical data to reduce customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = spark.read.csv('train.csv',header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_data = spark.read.csv('test.csv',header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+--------+--------+--------+--------+--------+--------+--------+-------+-------+-------+--------+--------+--------+--------+-------+\n",
      "| id|cat1|cat2|cat3|cat4|cat5|cat6|cat7|cat8|cat9|cat10|cat11|cat12|cat13|cat14|cat15|cat16|cat17|cat18|cat19|cat20|cat21|cat22|cat23|cat24|cat25|cat26|cat27|cat28|cat29|cat30|cat31|cat32|cat33|cat34|cat35|cat36|cat37|cat38|cat39|cat40|cat41|cat42|cat43|cat44|cat45|cat46|cat47|cat48|cat49|cat50|cat51|cat52|cat53|cat54|cat55|cat56|cat57|cat58|cat59|cat60|cat61|cat62|cat63|cat64|cat65|cat66|cat67|cat68|cat69|cat70|cat71|cat72|cat73|cat74|cat75|cat76|cat77|cat78|cat79|cat80|cat81|cat82|cat83|cat84|cat85|cat86|cat87|cat88|cat89|cat90|cat91|cat92|cat93|cat94|cat95|cat96|cat97|cat98|cat99|cat100|cat101|cat102|cat103|cat104|cat105|cat106|cat107|cat108|cat109|cat110|cat111|cat112|cat113|cat114|cat115|cat116|   cont1|   cont2|   cont3|   cont4|   cont5|   cont6|   cont7|  cont8|  cont9| cont10|  cont11|  cont12|  cont13|  cont14|   loss|\n",
      "+---+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+--------+--------+--------+--------+--------+--------+--------+-------+-------+-------+--------+--------+--------+--------+-------+\n",
      "|  1|   A|   B|   A|   B|   A|   A|   A|   A|   B|    A|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    D|    B|    B|    D|    D|    B|    D|    C|    B|    D|    B|    A|    A|    A|    A|    A|    D|    B|    C|    E|    A|    C|    T|     B|     G|     A|     A|     I|     E|     G|     J|     G|    BU|    BC|     C|    AS|     S|     A|     O|    LB|  0.7263|0.245921|0.187583|0.789639|0.310061|0.718367| 0.33506| 0.3026|0.67135| 0.8351|0.569745|0.594646|0.822493|0.714843|2213.18|\n",
      "|  2|   A|   B|   A|   A|   A|   A|   A|   A|   B|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    D|    B|    B|    D|    D|    A|    B|    C|    B|    D|    B|    A|    A|    A|    A|    A|    D|    D|    C|    E|    E|    D|    T|     L|     F|     A|     A|     E|     E|     I|     K|     K|    BI|    CQ|     A|    AV|    BM|     A|     O|    DP|0.330514|0.737068|0.592681|0.614134|0.885834|0.438917|0.436585|0.60087|0.35127|0.43919|0.338312|0.366307|0.611431|0.304496| 1283.6|\n",
      "|  5|   A|   B|   A|   A|   B|   A|   A|   A|   B|    B|    B|    B|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    D|    B|    B|    B|    D|    B|    D|    C|    B|    B|    B|    A|    A|    A|    A|    A|    D|    D|    C|    E|    E|    A|    D|     L|     O|     A|     B|     E|     F|     H|     F|     A|    AB|    DK|     A|     C|    AF|     A|     I|    GK|0.261841|0.358319|0.484196|0.236924|0.397069|0.289648|0.315545| 0.2732|0.26076|0.32446|0.381398|0.373424|0.195709|0.774425|3005.09|\n",
      "| 10|   B|   B|   A|   B|   A|   A|   A|   A|   B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    A|    A|    D|    B|    B|    D|    D|    D|    B|    C|    B|    D|    B|    A|    A|    A|    A|    A|    D|    D|    C|    E|    E|    D|    T|     I|     D|     A|     A|     E|     E|     I|     K|     K|    BI|    CS|     C|     N|    AE|     A|     O|    DJ|0.321594|0.555782|0.527991|0.373816|0.422268|0.440945|0.391128|0.31796|0.32128|0.44467|0.327915| 0.32157|0.605077|0.602642| 939.85|\n",
      "| 11|   A|   B|   A|   B|   A|   A|   A|   A|   B|    B|    A|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    A|    B|    A|    A|    A|    A|    D|    B|    D|    B|    D|    B|    B|    C|    B|    B|    C|    A|    A|    A|    B|    H|    D|    B|    D|    E|    E|    A|    P|     F|     J|     A|     A|     D|     E|     K|     G|     B|     H|     C|     C|     Y|    BM|     A|     K|    CK|0.273204| 0.15999|0.527991|0.473202|0.704268|0.178193|0.247408|0.24564|0.22089| 0.2123|0.204687|0.202213|0.246011|0.432606|2763.85|\n",
      "+---+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+--------+--------+--------+--------+--------+--------+--------+-------+-------+-------+--------+--------+--------+--------+-------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_data.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- id: integer (nullable = true)\n",
      " |-- cat1: string (nullable = true)\n",
      " |-- cat2: string (nullable = true)\n",
      " |-- cat3: string (nullable = true)\n",
      " |-- cat4: string (nullable = true)\n",
      " |-- cat5: string (nullable = true)\n",
      " |-- cat6: string (nullable = true)\n",
      " |-- cat7: string (nullable = true)\n",
      " |-- cat8: string (nullable = true)\n",
      " |-- cat9: string (nullable = true)\n",
      " |-- cat10: string (nullable = true)\n",
      " |-- cat11: string (nullable = true)\n",
      " |-- cat12: string (nullable = true)\n",
      " |-- cat13: string (nullable = true)\n",
      " |-- cat14: string (nullable = true)\n",
      " |-- cat15: string (nullable = true)\n",
      " |-- cat16: string (nullable = true)\n",
      " |-- cat17: string (nullable = true)\n",
      " |-- cat18: string (nullable = true)\n",
      " |-- cat19: string (nullable = true)\n",
      " |-- cat20: string (nullable = true)\n",
      " |-- cat21: string (nullable = true)\n",
      " |-- cat22: string (nullable = true)\n",
      " |-- cat23: string (nullable = true)\n",
      " |-- cat24: string (nullable = true)\n",
      " |-- cat25: string (nullable = true)\n",
      " |-- cat26: string (nullable = true)\n",
      " |-- cat27: string (nullable = true)\n",
      " |-- cat28: string (nullable = true)\n",
      " |-- cat29: string (nullable = true)\n",
      " |-- cat30: string (nullable = true)\n",
      " |-- cat31: string (nullable = true)\n",
      " |-- cat32: string (nullable = true)\n",
      " |-- cat33: string (nullable = true)\n",
      " |-- cat34: string (nullable = true)\n",
      " |-- cat35: string (nullable = true)\n",
      " |-- cat36: string (nullable = true)\n",
      " |-- cat37: string (nullable = true)\n",
      " |-- cat38: string (nullable = true)\n",
      " |-- cat39: string (nullable = true)\n",
      " |-- cat40: string (nullable = true)\n",
      " |-- cat41: string (nullable = true)\n",
      " |-- cat42: string (nullable = true)\n",
      " |-- cat43: string (nullable = true)\n",
      " |-- cat44: string (nullable = true)\n",
      " |-- cat45: string (nullable = true)\n",
      " |-- cat46: string (nullable = true)\n",
      " |-- cat47: string (nullable = true)\n",
      " |-- cat48: string (nullable = true)\n",
      " |-- cat49: string (nullable = true)\n",
      " |-- cat50: string (nullable = true)\n",
      " |-- cat51: string (nullable = true)\n",
      " |-- cat52: string (nullable = true)\n",
      " |-- cat53: string (nullable = true)\n",
      " |-- cat54: string (nullable = true)\n",
      " |-- cat55: string (nullable = true)\n",
      " |-- cat56: string (nullable = true)\n",
      " |-- cat57: string (nullable = true)\n",
      " |-- cat58: string (nullable = true)\n",
      " |-- cat59: string (nullable = true)\n",
      " |-- cat60: string (nullable = true)\n",
      " |-- cat61: string (nullable = true)\n",
      " |-- cat62: string (nullable = true)\n",
      " |-- cat63: string (nullable = true)\n",
      " |-- cat64: string (nullable = true)\n",
      " |-- cat65: string (nullable = true)\n",
      " |-- cat66: string (nullable = true)\n",
      " |-- cat67: string (nullable = true)\n",
      " |-- cat68: string (nullable = true)\n",
      " |-- cat69: string (nullable = true)\n",
      " |-- cat70: string (nullable = true)\n",
      " |-- cat71: string (nullable = true)\n",
      " |-- cat72: string (nullable = true)\n",
      " |-- cat73: string (nullable = true)\n",
      " |-- cat74: string (nullable = true)\n",
      " |-- cat75: string (nullable = true)\n",
      " |-- cat76: string (nullable = true)\n",
      " |-- cat77: string (nullable = true)\n",
      " |-- cat78: string (nullable = true)\n",
      " |-- cat79: string (nullable = true)\n",
      " |-- cat80: string (nullable = true)\n",
      " |-- cat81: string (nullable = true)\n",
      " |-- cat82: string (nullable = true)\n",
      " |-- cat83: string (nullable = true)\n",
      " |-- cat84: string (nullable = true)\n",
      " |-- cat85: string (nullable = true)\n",
      " |-- cat86: string (nullable = true)\n",
      " |-- cat87: string (nullable = true)\n",
      " |-- cat88: string (nullable = true)\n",
      " |-- cat89: string (nullable = true)\n",
      " |-- cat90: string (nullable = true)\n",
      " |-- cat91: string (nullable = true)\n",
      " |-- cat92: string (nullable = true)\n",
      " |-- cat93: string (nullable = true)\n",
      " |-- cat94: string (nullable = true)\n",
      " |-- cat95: string (nullable = true)\n",
      " |-- cat96: string (nullable = true)\n",
      " |-- cat97: string (nullable = true)\n",
      " |-- cat98: string (nullable = true)\n",
      " |-- cat99: string (nullable = true)\n",
      " |-- cat100: string (nullable = true)\n",
      " |-- cat101: string (nullable = true)\n",
      " |-- cat102: string (nullable = true)\n",
      " |-- cat103: string (nullable = true)\n",
      " |-- cat104: string (nullable = true)\n",
      " |-- cat105: string (nullable = true)\n",
      " |-- cat106: string (nullable = true)\n",
      " |-- cat107: string (nullable = true)\n",
      " |-- cat108: string (nullable = true)\n",
      " |-- cat109: string (nullable = true)\n",
      " |-- cat110: string (nullable = true)\n",
      " |-- cat111: string (nullable = true)\n",
      " |-- cat112: string (nullable = true)\n",
      " |-- cat113: string (nullable = true)\n",
      " |-- cat114: string (nullable = true)\n",
      " |-- cat115: string (nullable = true)\n",
      " |-- cat116: string (nullable = true)\n",
      " |-- cont1: double (nullable = true)\n",
      " |-- cont2: double (nullable = true)\n",
      " |-- cont3: double (nullable = true)\n",
      " |-- cont4: double (nullable = true)\n",
      " |-- cont5: double (nullable = true)\n",
      " |-- cont6: double (nullable = true)\n",
      " |-- cont7: double (nullable = true)\n",
      " |-- cont8: double (nullable = true)\n",
      " |-- cont9: double (nullable = true)\n",
      " |-- cont10: double (nullable = true)\n",
      " |-- cont11: double (nullable = true)\n",
      " |-- cont12: double (nullable = true)\n",
      " |-- cont13: double (nullable = true)\n",
      " |-- cont14: double (nullable = true)\n",
      " |-- loss: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "188318"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = train_data.withColumnRenamed(\"loss\",\"label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "[trainingData, validationData] = train_data.randomSplit([0.7, 0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[id: int, cat1: string, cat2: string, cat3: string, cat4: string, cat5: string, cat6: string, cat7: string, cat8: string, cat9: string, cat10: string, cat11: string, cat12: string, cat13: string, cat14: string, cat15: string, cat16: string, cat17: string, cat18: string, cat19: string, cat20: string, cat21: string, cat22: string, cat23: string, cat24: string, cat25: string, cat26: string, cat27: string, cat28: string, cat29: string, cat30: string, cat31: string, cat32: string, cat33: string, cat34: string, cat35: string, cat36: string, cat37: string, cat38: string, cat39: string, cat40: string, cat41: string, cat42: string, cat43: string, cat44: string, cat45: string, cat46: string, cat47: string, cat48: string, cat49: string, cat50: string, cat51: string, cat52: string, cat53: string, cat54: string, cat55: string, cat56: string, cat57: string, cat58: string, cat59: string, cat60: string, cat61: string, cat62: string, cat63: string, cat64: string, cat65: string, cat66: string, cat67: string, cat68: string, cat69: string, cat70: string, cat71: string, cat72: string, cat73: string, cat74: string, cat75: string, cat76: string, cat77: string, cat78: string, cat79: string, cat80: string, cat81: string, cat82: string, cat83: string, cat84: string, cat85: string, cat86: string, cat87: string, cat88: string, cat89: string, cat90: string, cat91: string, cat92: string, cat93: string, cat94: string, cat95: string, cat96: string, cat97: string, cat98: string, cat99: string, cat100: string, cat101: string, cat102: string, cat103: string, cat104: string, cat105: string, cat106: string, cat107: string, cat108: string, cat109: string, cat110: string, cat111: string, cat112: string, cat113: string, cat114: string, cat115: string, cat116: string, cont1: double, cont2: double, cont3: double, cont4: double, cont5: double, cont6: double, cont7: double, cont8: double, cont9: double, cont10: double, cont11: double, cont12: double, cont13: double, cont14: double, label: double]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Prevent loading of data again & again\n",
    "trainingData.cache()\n",
    "validationData.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a pipeline\n",
    "* There are many categorical cols, starts with cat* in this data\n",
    "* All cont data starts with cont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "columns = train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Gets all the categorical cols\n",
    "cat_cols = list(filter(lambda s: s.startswith(\"cat\"), columns))\n",
    "#cat_cols = cat_cols[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cont_cols = list(filter(lambda s: s.startswith(\"cont\"), columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#We have to create list of stringindexer, one stringindexer for different col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stringIndexList = []\n",
    "import pyspark.ml.feature as ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for col in cat_cols:\n",
    "    stringIndexList.append(ft.StringIndexer(inputCol=col, outputCol='idx'+col[3:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "a = stringIndexList[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'idx4'"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.getOutputCol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "116"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stringIndexList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "idx_cat_cols = list(map(lambda s: s.replace('cat','idx'), cat_cols))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline, PipelineModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assembler = ft.VectorAssembler(inputCols = idx_cat_cols+cont_cols, outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import RandomForestRegressor\n",
    "estimator = RandomForestRegressor(numTrees=3,maxDepth=5, maxBins=350, featuresCol=\"features\", labelCol=\"label\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "stages = []\n",
    "stages.extend(stringIndexList)\n",
    "stages.append(assembler)\n",
    "stages.append(estimator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline(stages=stages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = pipeline.fit(trainingData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "out = model.transform(validationData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------------------+\n",
      "|   label|        prediction|\n",
      "+--------+------------------+\n",
      "| 3005.09| 4874.184676703838|\n",
      "| 10280.2|6939.8000125114895|\n",
      "| 1193.05|2069.9152463290293|\n",
      "| 6609.32| 7187.607941571394|\n",
      "| 1528.73|2251.9649380259784|\n",
      "|11673.03| 5226.869996102717|\n",
      "|11248.08|10115.243011490076|\n",
      "|  876.78| 2016.257098309675|\n",
      "| 4822.07| 3770.916919062289|\n",
      "| 2809.74| 5226.869996102717|\n",
      "|  786.26| 2199.239868919549|\n",
      "| 1028.04| 2199.239868919549|\n",
      "| 3976.75|1857.6065456807844|\n",
      "| 7484.37| 3770.916919062289|\n",
      "| 3343.49| 2199.239868919549|\n",
      "| 1997.58| 6550.609946302468|\n",
      "|  817.74| 2199.239868919549|\n",
      "| 2585.55| 2199.239868919549|\n",
      "| 6213.76| 2199.239868919549|\n",
      "|  487.17| 3140.064214062986|\n",
      "+--------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "out['label','prediction'].show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluator = RegressionEvaluator(labelCol=\"label\", \n",
    "                                predictionCol=\"prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#rev.evaluate(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}