Week 4/code/.ipynb_checkpoints/Churn Ensembles Example in Python-PM class-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensembles Tuning and CV\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Like R, Python uses packages in data mining/machine learning. The 3 mose common ones are Pandas (manipulation), Scikit Learn (machine learning) and Matplotlit (graphics)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/mpgartland/Documents/Courses/Predictive Models/Pred_Models_git/Week 2'"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Add packages\n",
    "%matplotlib inline \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, KFold\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import roc_curve, auc, confusion_matrix, classification_report\n",
    "from sklearn import tree \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "import scipy.stats as ss\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "import time\n",
    "from operator import itemgetter\n",
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/mpgartland/Documents/Courses/Predictive Models/Pred_Models_git/Week 2\n"
     ]
    }
   ],
   "source": [
    "cd '/Users/mpgartland/Documents/Courses/Predictive Models/Pred_Models_git/Week 3/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in Data\n",
    "# Churn Calls Data\n",
    "This is a Pandas operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state</th>\n",
       "      <th>account_length</th>\n",
       "      <th>area_code</th>\n",
       "      <th>international_plan</th>\n",
       "      <th>voice_mail_plan</th>\n",
       "      <th>number_vmail_messages</th>\n",
       "      <th>total_day_minutes</th>\n",
       "      <th>total_day_calls</th>\n",
       "      <th>total_day_charge</th>\n",
       "      <th>total_eve_minutes</th>\n",
       "      <th>total_eve_calls</th>\n",
       "      <th>total_eve_charge</th>\n",
       "      <th>total_night_minutes</th>\n",
       "      <th>total_night_calls</th>\n",
       "      <th>total_night_charge</th>\n",
       "      <th>total_intl_minutes</th>\n",
       "      <th>total_intl_calls</th>\n",
       "      <th>total_intl_charge</th>\n",
       "      <th>number_customer_service_calls</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AK</td>\n",
       "      <td>1</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>175.2</td>\n",
       "      <td>74</td>\n",
       "      <td>29.78</td>\n",
       "      <td>151.7</td>\n",
       "      <td>79</td>\n",
       "      <td>12.89</td>\n",
       "      <td>230.5</td>\n",
       "      <td>109</td>\n",
       "      <td>10.37</td>\n",
       "      <td>5.3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.43</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>30</td>\n",
       "      <td>146.3</td>\n",
       "      <td>128</td>\n",
       "      <td>24.87</td>\n",
       "      <td>162.5</td>\n",
       "      <td>80</td>\n",
       "      <td>13.81</td>\n",
       "      <td>129.3</td>\n",
       "      <td>109</td>\n",
       "      <td>5.82</td>\n",
       "      <td>14.5</td>\n",
       "      <td>6</td>\n",
       "      <td>3.92</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>19</td>\n",
       "      <td>171.9</td>\n",
       "      <td>96</td>\n",
       "      <td>29.22</td>\n",
       "      <td>198.4</td>\n",
       "      <td>111</td>\n",
       "      <td>16.86</td>\n",
       "      <td>321.7</td>\n",
       "      <td>76</td>\n",
       "      <td>14.48</td>\n",
       "      <td>10.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.84</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AK</td>\n",
       "      <td>41</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>159.3</td>\n",
       "      <td>66</td>\n",
       "      <td>27.08</td>\n",
       "      <td>125.9</td>\n",
       "      <td>75</td>\n",
       "      <td>10.70</td>\n",
       "      <td>261.9</td>\n",
       "      <td>76</td>\n",
       "      <td>11.79</td>\n",
       "      <td>11.1</td>\n",
       "      <td>5</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AK</td>\n",
       "      <td>42</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>129</td>\n",
       "      <td>29.07</td>\n",
       "      <td>183.9</td>\n",
       "      <td>96</td>\n",
       "      <td>15.63</td>\n",
       "      <td>130.2</td>\n",
       "      <td>90</td>\n",
       "      <td>5.86</td>\n",
       "      <td>4.6</td>\n",
       "      <td>6</td>\n",
       "      <td>1.24</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AK</td>\n",
       "      <td>48</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>37</td>\n",
       "      <td>211.7</td>\n",
       "      <td>115</td>\n",
       "      <td>35.99</td>\n",
       "      <td>159.9</td>\n",
       "      <td>84</td>\n",
       "      <td>13.59</td>\n",
       "      <td>144.1</td>\n",
       "      <td>80</td>\n",
       "      <td>6.48</td>\n",
       "      <td>12.2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.29</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AK</td>\n",
       "      <td>50</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>183.6</td>\n",
       "      <td>107</td>\n",
       "      <td>31.21</td>\n",
       "      <td>58.6</td>\n",
       "      <td>118</td>\n",
       "      <td>4.98</td>\n",
       "      <td>202.6</td>\n",
       "      <td>99</td>\n",
       "      <td>9.12</td>\n",
       "      <td>8.7</td>\n",
       "      <td>3</td>\n",
       "      <td>2.35</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AK</td>\n",
       "      <td>51</td>\n",
       "      <td>area_code_510</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>12</td>\n",
       "      <td>135.8</td>\n",
       "      <td>60</td>\n",
       "      <td>23.09</td>\n",
       "      <td>200.6</td>\n",
       "      <td>134</td>\n",
       "      <td>17.05</td>\n",
       "      <td>192.4</td>\n",
       "      <td>98</td>\n",
       "      <td>8.66</td>\n",
       "      <td>12.3</td>\n",
       "      <td>7</td>\n",
       "      <td>3.32</td>\n",
       "      <td>2</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>104</td>\n",
       "      <td>36.89</td>\n",
       "      <td>152.3</td>\n",
       "      <td>83</td>\n",
       "      <td>12.95</td>\n",
       "      <td>134.3</td>\n",
       "      <td>109</td>\n",
       "      <td>6.04</td>\n",
       "      <td>11.8</td>\n",
       "      <td>4</td>\n",
       "      <td>3.19</td>\n",
       "      <td>2</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>24</td>\n",
       "      <td>170.9</td>\n",
       "      <td>71</td>\n",
       "      <td>29.05</td>\n",
       "      <td>201.4</td>\n",
       "      <td>80</td>\n",
       "      <td>17.12</td>\n",
       "      <td>159.0</td>\n",
       "      <td>124</td>\n",
       "      <td>7.15</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.11</td>\n",
       "      <td>2</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  state  account_length      area_code international_plan voice_mail_plan  \\\n",
       "0    AK               1  area_code_408                 no              no   \n",
       "1    AK              36  area_code_408                 no             yes   \n",
       "2    AK              36  area_code_415                yes             yes   \n",
       "3    AK              41  area_code_415                 no              no   \n",
       "4    AK              42  area_code_415                 no              no   \n",
       "5    AK              48  area_code_415                 no             yes   \n",
       "6    AK              50  area_code_408                 no              no   \n",
       "7    AK              51  area_code_510                yes             yes   \n",
       "8    AK              52  area_code_408                 no              no   \n",
       "9    AK              52  area_code_415                 no             yes   \n",
       "\n",
       "   number_vmail_messages  total_day_minutes  total_day_calls  \\\n",
       "0                      0              175.2               74   \n",
       "1                     30              146.3              128   \n",
       "2                     19              171.9               96   \n",
       "3                      0              159.3               66   \n",
       "4                      0              171.0              129   \n",
       "5                     37              211.7              115   \n",
       "6                      0              183.6              107   \n",
       "7                     12              135.8               60   \n",
       "8                      0              217.0              104   \n",
       "9                     24              170.9               71   \n",
       "\n",
       "   total_day_charge  total_eve_minutes  total_eve_calls  total_eve_charge  \\\n",
       "0             29.78              151.7               79             12.89   \n",
       "1             24.87              162.5               80             13.81   \n",
       "2             29.22              198.4              111             16.86   \n",
       "3             27.08              125.9               75             10.70   \n",
       "4             29.07              183.9               96             15.63   \n",
       "5             35.99              159.9               84             13.59   \n",
       "6             31.21               58.6              118              4.98   \n",
       "7             23.09              200.6              134             17.05   \n",
       "8             36.89              152.3               83             12.95   \n",
       "9             29.05              201.4               80             17.12   \n",
       "\n",
       "   total_night_minutes  total_night_calls  total_night_charge  \\\n",
       "0                230.5                109               10.37   \n",
       "1                129.3                109                5.82   \n",
       "2                321.7                 76               14.48   \n",
       "3                261.9                 76               11.79   \n",
       "4                130.2                 90                5.86   \n",
       "5                144.1                 80                6.48   \n",
       "6                202.6                 99                9.12   \n",
       "7                192.4                 98                8.66   \n",
       "8                134.3                109                6.04   \n",
       "9                159.0                124                7.15   \n",
       "\n",
       "   total_intl_minutes  total_intl_calls  total_intl_charge  \\\n",
       "0                 5.3                 3               1.43   \n",
       "1                14.5                 6               3.92   \n",
       "2                10.5                 1               2.84   \n",
       "3                11.1                 5               3.00   \n",
       "4                 4.6                 6               1.24   \n",
       "5                12.2                 1               3.29   \n",
       "6                 8.7                 3               2.35   \n",
       "7                12.3                 7               3.32   \n",
       "8                11.8                 4               3.19   \n",
       "9                 4.1                 5               1.11   \n",
       "\n",
       "   number_customer_service_calls churn  \n",
       "0                              1    no  \n",
       "1                              0    no  \n",
       "2                              1   yes  \n",
       "3                              1    no  \n",
       "4                              0    no  \n",
       "5                              1    no  \n",
       "6                              1    no  \n",
       "7                              2    no  \n",
       "8                              2    no  \n",
       "9                              2    no  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import data\n",
    "df = pd.read_csv(\"data/Churn_Calls.csv\", sep=',')\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['state', 'account_length', 'area_code', 'international_plan',\n",
      "       'voice_mail_plan', 'number_vmail_messages', 'total_day_minutes',\n",
      "       'total_day_calls', 'total_day_charge', 'total_eve_minutes',\n",
      "       'total_eve_calls', 'total_eve_charge', 'total_night_minutes',\n",
      "       'total_night_calls', 'total_night_charge', 'total_intl_minutes',\n",
      "       'total_intl_calls', 'total_intl_charge',\n",
      "       'number_customer_service_calls', 'churn'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# See each collum name\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 20)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Target\n",
    "In this step I took the target variable and moved it to the first collum. I aslo made a reference to it called targetName. This just helps me with some below steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churn</th>\n",
       "      <th>state</th>\n",
       "      <th>account_length</th>\n",
       "      <th>area_code</th>\n",
       "      <th>international_plan</th>\n",
       "      <th>voice_mail_plan</th>\n",
       "      <th>number_vmail_messages</th>\n",
       "      <th>total_day_minutes</th>\n",
       "      <th>total_day_calls</th>\n",
       "      <th>total_day_charge</th>\n",
       "      <th>total_eve_minutes</th>\n",
       "      <th>total_eve_calls</th>\n",
       "      <th>total_eve_charge</th>\n",
       "      <th>total_night_minutes</th>\n",
       "      <th>total_night_calls</th>\n",
       "      <th>total_night_charge</th>\n",
       "      <th>total_intl_minutes</th>\n",
       "      <th>total_intl_calls</th>\n",
       "      <th>total_intl_charge</th>\n",
       "      <th>number_customer_service_calls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>1</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>175.2</td>\n",
       "      <td>74</td>\n",
       "      <td>29.78</td>\n",
       "      <td>151.7</td>\n",
       "      <td>79</td>\n",
       "      <td>12.89</td>\n",
       "      <td>230.5</td>\n",
       "      <td>109</td>\n",
       "      <td>10.37</td>\n",
       "      <td>5.3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.43</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>30</td>\n",
       "      <td>146.3</td>\n",
       "      <td>128</td>\n",
       "      <td>24.87</td>\n",
       "      <td>162.5</td>\n",
       "      <td>80</td>\n",
       "      <td>13.81</td>\n",
       "      <td>129.3</td>\n",
       "      <td>109</td>\n",
       "      <td>5.82</td>\n",
       "      <td>14.5</td>\n",
       "      <td>6</td>\n",
       "      <td>3.92</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yes</td>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>19</td>\n",
       "      <td>171.9</td>\n",
       "      <td>96</td>\n",
       "      <td>29.22</td>\n",
       "      <td>198.4</td>\n",
       "      <td>111</td>\n",
       "      <td>16.86</td>\n",
       "      <td>321.7</td>\n",
       "      <td>76</td>\n",
       "      <td>14.48</td>\n",
       "      <td>10.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.84</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>41</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>159.3</td>\n",
       "      <td>66</td>\n",
       "      <td>27.08</td>\n",
       "      <td>125.9</td>\n",
       "      <td>75</td>\n",
       "      <td>10.70</td>\n",
       "      <td>261.9</td>\n",
       "      <td>76</td>\n",
       "      <td>11.79</td>\n",
       "      <td>11.1</td>\n",
       "      <td>5</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>42</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>129</td>\n",
       "      <td>29.07</td>\n",
       "      <td>183.9</td>\n",
       "      <td>96</td>\n",
       "      <td>15.63</td>\n",
       "      <td>130.2</td>\n",
       "      <td>90</td>\n",
       "      <td>5.86</td>\n",
       "      <td>4.6</td>\n",
       "      <td>6</td>\n",
       "      <td>1.24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>48</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>37</td>\n",
       "      <td>211.7</td>\n",
       "      <td>115</td>\n",
       "      <td>35.99</td>\n",
       "      <td>159.9</td>\n",
       "      <td>84</td>\n",
       "      <td>13.59</td>\n",
       "      <td>144.1</td>\n",
       "      <td>80</td>\n",
       "      <td>6.48</td>\n",
       "      <td>12.2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.29</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>50</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>183.6</td>\n",
       "      <td>107</td>\n",
       "      <td>31.21</td>\n",
       "      <td>58.6</td>\n",
       "      <td>118</td>\n",
       "      <td>4.98</td>\n",
       "      <td>202.6</td>\n",
       "      <td>99</td>\n",
       "      <td>9.12</td>\n",
       "      <td>8.7</td>\n",
       "      <td>3</td>\n",
       "      <td>2.35</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>51</td>\n",
       "      <td>area_code_510</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>12</td>\n",
       "      <td>135.8</td>\n",
       "      <td>60</td>\n",
       "      <td>23.09</td>\n",
       "      <td>200.6</td>\n",
       "      <td>134</td>\n",
       "      <td>17.05</td>\n",
       "      <td>192.4</td>\n",
       "      <td>98</td>\n",
       "      <td>8.66</td>\n",
       "      <td>12.3</td>\n",
       "      <td>7</td>\n",
       "      <td>3.32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>104</td>\n",
       "      <td>36.89</td>\n",
       "      <td>152.3</td>\n",
       "      <td>83</td>\n",
       "      <td>12.95</td>\n",
       "      <td>134.3</td>\n",
       "      <td>109</td>\n",
       "      <td>6.04</td>\n",
       "      <td>11.8</td>\n",
       "      <td>4</td>\n",
       "      <td>3.19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>24</td>\n",
       "      <td>170.9</td>\n",
       "      <td>71</td>\n",
       "      <td>29.05</td>\n",
       "      <td>201.4</td>\n",
       "      <td>80</td>\n",
       "      <td>17.12</td>\n",
       "      <td>159.0</td>\n",
       "      <td>124</td>\n",
       "      <td>7.15</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  churn state  account_length      area_code international_plan  \\\n",
       "0    no    AK               1  area_code_408                 no   \n",
       "1    no    AK              36  area_code_408                 no   \n",
       "2   yes    AK              36  area_code_415                yes   \n",
       "3    no    AK              41  area_code_415                 no   \n",
       "4    no    AK              42  area_code_415                 no   \n",
       "5    no    AK              48  area_code_415                 no   \n",
       "6    no    AK              50  area_code_408                 no   \n",
       "7    no    AK              51  area_code_510                yes   \n",
       "8    no    AK              52  area_code_408                 no   \n",
       "9    no    AK              52  area_code_415                 no   \n",
       "\n",
       "  voice_mail_plan  number_vmail_messages  total_day_minutes  total_day_calls  \\\n",
       "0              no                      0              175.2               74   \n",
       "1             yes                     30              146.3              128   \n",
       "2             yes                     19              171.9               96   \n",
       "3              no                      0              159.3               66   \n",
       "4              no                      0              171.0              129   \n",
       "5             yes                     37              211.7              115   \n",
       "6              no                      0              183.6              107   \n",
       "7             yes                     12              135.8               60   \n",
       "8              no                      0              217.0              104   \n",
       "9             yes                     24              170.9               71   \n",
       "\n",
       "   total_day_charge  total_eve_minutes  total_eve_calls  total_eve_charge  \\\n",
       "0             29.78              151.7               79             12.89   \n",
       "1             24.87              162.5               80             13.81   \n",
       "2             29.22              198.4              111             16.86   \n",
       "3             27.08              125.9               75             10.70   \n",
       "4             29.07              183.9               96             15.63   \n",
       "5             35.99              159.9               84             13.59   \n",
       "6             31.21               58.6              118              4.98   \n",
       "7             23.09              200.6              134             17.05   \n",
       "8             36.89              152.3               83             12.95   \n",
       "9             29.05              201.4               80             17.12   \n",
       "\n",
       "   total_night_minutes  total_night_calls  total_night_charge  \\\n",
       "0                230.5                109               10.37   \n",
       "1                129.3                109                5.82   \n",
       "2                321.7                 76               14.48   \n",
       "3                261.9                 76               11.79   \n",
       "4                130.2                 90                5.86   \n",
       "5                144.1                 80                6.48   \n",
       "6                202.6                 99                9.12   \n",
       "7                192.4                 98                8.66   \n",
       "8                134.3                109                6.04   \n",
       "9                159.0                124                7.15   \n",
       "\n",
       "   total_intl_minutes  total_intl_calls  total_intl_charge  \\\n",
       "0                 5.3                 3               1.43   \n",
       "1                14.5                 6               3.92   \n",
       "2                10.5                 1               2.84   \n",
       "3                11.1                 5               3.00   \n",
       "4                 4.6                 6               1.24   \n",
       "5                12.2                 1               3.29   \n",
       "6                 8.7                 3               2.35   \n",
       "7                12.3                 7               3.32   \n",
       "8                11.8                 4               3.19   \n",
       "9                 4.1                 5               1.11   \n",
       "\n",
       "   number_customer_service_calls  \n",
       "0                              1  \n",
       "1                              0  \n",
       "2                              1  \n",
       "3                              1  \n",
       "4                              0  \n",
       "5                              1  \n",
       "6                              1  \n",
       "7                              2  \n",
       "8                              2  \n",
       "9                              2  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# designate target variable name\n",
    "targetName = 'churn'\n",
    "# move target variable into first column\n",
    "targetSeries = df[targetName]\n",
    "del df[targetName]\n",
    "df.insert(0, targetName, targetSeries)\n",
    "expected=targetName\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#EDA\n",
    "Just a touch of EDA. This is the distribution of the target. As you can see, the datset is imbalanced and the target class of interest \"yes\" is in the minority (a common occurance in classification)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x113fcc390>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAERCAYAAACdPxtnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADrVJREFUeJzt3X+s3fVdx/Hna2VjRGVQuTbYEktmNSnoBjSVhGkIOOnC\nYtFspEuUaggkghlLjKaYGaJJk6rxR4hCrBtS1FirLqOBkQW7gTGT1QtjY+2sVKApFWhhYjddOlve\n/nE+ZGd3t7v3wu39wv08H8nJ+Z7P+X6/93OSe/O83+/3nHtTVUiS+vSWoScgSRqOEZCkjhkBSeqY\nEZCkjhkBSeqYEZCkjhkBSeqYEZCkjhkBSerYaUNPYCbnnHNOrVy5cuhpSNKbyqOPPvpiVU3MtN4b\nPgIrV65kcnJy6GlI0ptKkgOzWc/TQZLUMSMgSR0zApLUMSMgSR0zApLUMSMgSR0zApLUMSMgSR17\nw39Y7M1i5ab7h57CovHMlquHnoLUDY8EJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCS\nOmYEJKljRkCSOjbrCCRZkuQLSe5rj5cmeTDJk+3+7LF1b02yP8m+JFeNjV+S5In23O1JMr8vR5I0\nF3M5ErgF+MrY403ArqpaBexqj0myGtgAXACsA+5IsqRtcydwA7Cq3da9rtlLkl6XWUUgyQrgauBj\nY8PrgW1teRtwzdj49qo6VlVPA/uBtUnOBc6sqkeqqoB7xraRJA1gtkcCfwz8BvDK2NiyqnquLT8P\nLGvLy4GDY+s928aWt+Wp45KkgcwYgSTvBw5X1aMnW6f9Zl/zNakkNyaZTDJ55MiR+dqtJGmK2RwJ\nXAb8bJJngO3AFUn+CnihneKh3R9u6x8CzhvbfkUbO9SWp45/h6raWlVrqmrNxMTEHF6OJGkuZoxA\nVd1aVSuqaiWjC76fqapfAHYCG9tqG4F72/JOYEOS05Ocz+gC8O526uhokkvbu4KuG9tGkjSA1/Of\nxbYAO5JcDxwArgWoqj1JdgB7gePAzVV1om1zE3A3cAbwQLtJkgYypwhU1UPAQ235JeDKk6y3Gdg8\nzfgkcOFcJylJOjX8xLAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLH\njIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAk\ndcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwISFLHjIAkdcwI\nSFLHjIAkdcwISFLHjIAkdWzGCCR5e5LdSb6YZE+S327jS5M8mOTJdn/22Da3JtmfZF+Sq8bGL0ny\nRHvu9iQ5NS9LkjQbszkSOAZcUVXvAt4NrEtyKbAJ2FVVq4Bd7TFJVgMbgAuAdcAdSZa0fd0J3ACs\nard18/haJElzNGMEauTr7eFb262A9cC2Nr4NuKYtrwe2V9Wxqnoa2A+sTXIucGZVPVJVBdwzto0k\naQCzuiaQZEmSx4HDwINV9XlgWVU911Z5HljWlpcDB8c2f7aNLW/LU8clSQOZVQSq6kRVvRtYwei3\n+gunPF+Mjg7mRZIbk0wmmTxy5Mh87VaSNMWc3h1UVS8Dn2V0Lv+FdoqHdn+4rXYIOG9ssxVt7FBb\nnjo+3dfZWlVrqmrNxMTEXKYoSZqD2bw7aCLJWW35DOC9wL8BO4GNbbWNwL1teSewIcnpSc5ndAF4\ndzt1dDTJpe1dQdeNbSNJGsBps1jnXGBbe4fPW4AdVXVfkn8BdiS5HjgAXAtQVXuS7AD2AseBm6vq\nRNvXTcDdwBnAA+0mSRrIjBGoqi8BF00z/hJw5Um22QxsnmZ8ErjwO7eQJA3BTwxLUseMgCR1zAhI\nUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseM\ngCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1\nzAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1bMYI\nJDkvyWeT7E2yJ8ktbXxpkgeTPNnuzx7b5tYk+5PsS3LV2PglSZ5oz92eJKfmZUmSZmM2RwLHgV+r\nqtXApcDNSVYDm4BdVbUK2NUe057bAFwArAPuSLKk7etO4AZgVbutm8fXIkmaoxkjUFXPVdVjbflr\nwFeA5cB6YFtbbRtwTVteD2yvqmNV9TSwH1ib5FzgzKp6pKoKuGdsG0nSAOZ0TSDJSuAi4PPAsqp6\nrj31PLCsLS8HDo5t9mwbW96Wp45P93VuTDKZZPLIkSNzmaIkaQ5mHYEk3wv8A/CRqjo6/lz7zb7m\na1JVtbWq1lTVmomJifnarSRpillFIMlbGQXgr6vqE234hXaKh3Z/uI0fAs4b23xFGzvUlqeOS5IG\nMpt3BwX4OPCVqvrDsad2Ahvb8kbg3rHxDUlOT3I+owvAu9upo6NJLm37vG5sG0nSAE6bxTqXAb8I\nPJHk8Tb2m8AWYEeS64EDwLUAVbUnyQ5gL6N3Ft1cVSfadjcBdwNnAA+0myRpIDNGoKr+GTjZ+/mv\nPMk2m4HN04xPAhfOZYKSpFPHTwxLUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhI\nUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseM\ngCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1zAhIUseMgCR1\nzAhIUseMgCR1zAhIUseMgCR1zAhIUsdmjECSu5IcTvLlsbGlSR5M8mS7P3vsuVuT7E+yL8lVY+OX\nJHmiPXd7ksz/y5EkzcVsjgTuBtZNGdsE7KqqVcCu9pgkq4ENwAVtmzuSLGnb3AncAKxqt6n7lCQt\nsBkjUFX/BHx1yvB6YFtb3gZcMza+vaqOVdXTwH5gbZJzgTOr6pGqKuCesW0kSQN5rdcEllXVc235\neWBZW14OHBxb79k2trwtTx2fVpIbk0wmmTxy5MhrnKIkaSav+8Jw+82+5mEu4/vcWlVrqmrNxMTE\nfO5akjTmtUbghXaKh3Z/uI0fAs4bW29FGzvUlqeOS5IG9FojsBPY2JY3AveOjW9IcnqS8xldAN7d\nTh0dTXJpe1fQdWPbSJIGctpMKyT5G+By4JwkzwK3AVuAHUmuBw4A1wJU1Z4kO4C9wHHg5qo60XZ1\nE6N3Gp0BPNBukqQBzRiBqvrQSZ668iTrbwY2TzM+CVw4p9lJkk4pPzEsSR0zApLUMSMgSR0zApLU\nMSMgSR0zApLUMSMgSR0zApLUsRk/LCbpzW3lpvuHnsKi8syWq4eewrzySECSOmYEJKljRkCSOmYE\nJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKlj\nRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCSOmYEJKljRkCS\nOmYEJKljCx6BJOuS7EuyP8mmhf76kqRvWdAIJFkC/CnwPmA18KEkqxdyDpKkb1noI4G1wP6qeqqq\nvglsB9Yv8BwkSc1pC/z1lgMHxx4/C/zEd9tg3759XH755adyTvPi+adeGnoKi8blj/z+0FNYVPze\nnF+L7ftzoSMwK0luBG5sD7/+8MMP7xtyPovIOcCLQ09iJg8fnHkdLUp+f86vH5rNSgsdgUPAeWOP\nV7Sxb1NVW4GtCzWpXiSZrKo1Q89Dmo7fn8NY6GsC/wqsSnJ+krcBG4CdCzwHSVKzoEcCVXU8ya8C\nnwaWAHdV1Z6FnIMk6VsW/JpAVX0K+NRCf10BnmLTG5vfnwNIVQ09B0nSQPyzEZLUMSMgSR0zApLU\nMSMgaRBJPpjk+9ryR5N8IsnFQ8+rN0ZgkUvyjiR/lGSy3f4gyTuGnpcE/FZVfS3Je4CfBj4O3Dnw\nnLpjBBa/u4CjwLXtdhT4i0FnJI2caPdXA1ur6n7gbQPOp0u+RXSRS/J4Vb17pjFpoSW5j9GfjXkv\ncDHwDWB3Vb1r0Il1xiOBxe8b7XAbgCSXMfphk4Z2LaO/HnBVVb0MLAV+fdgp9ecN+VdENa9+Bdg2\ndh3gv4CNA85HAqCq/jfJYeA9wJPA8XavBeTpoEUuyenAB4B3AmcB/w1UVf3OoBNT95LcBqwBfrSq\nfiTJDwJ/V1WXDTy1rngksPjdC7wMPMY0f7ZbGtDPARcx+t6kqv7z1beMauEYgcVvRVWtG3oS0jS+\nWVWVpACSfM/QE+qRF4YXv88l+bGhJyFNY0eSPwPOSnID8I/Anw88p+54TWCRS7IX+GHgaeAYEEbX\nBH580Impe0k+DDwHrGX0ffnpqnpw2Fn1x9NBi9/7hp6AdBI/AHyY0TWBuxgdCWiBeSQgaTBJAvwM\n8MuM3im0A/h4Vf3HoBPriNcEJA2mRr+FPt9ux4Gzgb9P8nuDTqwjHglIGkSSW4DrgBeBjwGfrKr/\nS/IW4MmqeuegE+yE1wQkDWUp8PNVdWB8sKpeSfL+gebUHY8EJKljXhOQpI4ZAUnqmBGQppHk7iQf\nGHoe0qlmBKRTIMmSoecgzYYRkIAk1yX5UpIvJvnLNvxTST6X5KlXjwqSXN7+I9ar2/1Jkl9qy88k\n+d0kjwEfTPJQe7w7yb8n+ckFf2HSDIyAupfkAuCjwBXtXxve0p46l9E/PHk/sGWWu3upqi6uqu3t\n8WlVtRb4CHDbPE5bmhdGQIIrGP0zkxcBquqrbfyTVfVKVe0Fls1yX3875fEn2v2jwMrXO1FpvhkB\n6eSOjS2n3R/n239u3j5lm/85yT5O4Icz9QZkBCT4DKNz+N8PkGTpd1n3ALA6yelJzgKuXIgJSqeK\nv5moe1W1J8lm4OEkJ4AvfJd1DybZAXyZ0f9oOOm60puBfzZCkjrm6SBJ6pgRkKSOGQFJ6pgRkKSO\nGQFJ6pgRkKSOGQFJ6pgRkKSO/T9O4oLMmD/abgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10ffb6588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gb = df.groupby(targetName)\n",
    "targetEDA=gb[targetName].aggregate(len)\n",
    "plt.figure()\n",
    "targetEDA.plot(kind='bar', grid=False)\n",
    "plt.axhline(0, color='k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Preprocessing\n",
    "The below two steps are for preprocessing. The first cell changes the yes/no of the target to numeric. I needed to do this as some models require the target to be numeric. The second cell takes all the category features and creates dummies with them. This is stock code I have used for long time (and I did not write it). It is nice because it will take any dataframe of any size and handle categorial features. I do not have to change a single line in it. It can be used generically on bascially any dataframe. Saves a lot of time of coding each feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "le_dep = preprocessing.LabelEncoder()\n",
    "#to convert into numbers\n",
    "df['churn'] = le_dep.fit_transform(df['churn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# perform data transformation\n",
    "for col in df.columns[1:]:\n",
    "\tattName = col\n",
    "\tdType = df[col].dtype\n",
    "\tmissing = pd.isnull(df[col]).any()\n",
    "\tuniqueCount = len(df[attName].value_counts(normalize=False))\n",
    "\t# discretize (create dummies)\n",
    "\tif dType == object:\n",
    "\t\tdf = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)\n",
    "\t\tdel df[attName]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test/Train\n",
    "I split the data into a 60/40 train test. The features are stored in \"features_train\" and \"features_test\". The targets are in \"target_train\" and \"target_test\". I used a biggest test when I have an imbalanced set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split dataset into testing and training\n",
    "features_train, features_test, target_train, target_test = train_test_split(\n",
    "    df.ix[:,1:].values, df.ix[:,0].values, test_size=0.40, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just a view of the size of each test/train set.\n",
    "Note there are now 73 features, and the test set is imbalanced (14.6%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 73)\n",
      "(3000, 73)\n",
      "(2000,)\n",
      "(3000,)\n",
      "Percent of Target that is Yes 0.146\n"
     ]
    }
   ],
   "source": [
    "print(features_test.shape)\n",
    "print(features_train.shape)\n",
    "print(target_test.shape)\n",
    "print(target_train.shape)\n",
    "print(\"Percent of Target that is Yes\", target_test.mean())\n",
    "#data.groupby(['col1', 'col2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Models\n",
    "All the models are done in Sci-Kit Learn."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Decision Tree\n",
    "I created a decision tree from the data. The accurancy of the model was 921%, while the test data classified at 92%. However notice that the \"yes\" class (the class I am interested in) only properly classified at 74% (specificity) and .71 (recall). That is so-so. Again, not uncommon with imbalanced data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DT Accuracy Score 0.924\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "  Fail = no       0.95      0.96      0.96      1708\n",
      " Fail = yes       0.74      0.73      0.74       292\n",
      "\n",
      "avg / total       0.92      0.92      0.92      2000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Decision Tree train model\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf = clf.fit(features_train, target_train)\n",
    "#DT test model\n",
    "target_predicted_dt = clf.predict(features_test)\n",
    "print(\"DT Accuracy Score\", accuracy_score(target_test, target_predicted_dt))\n",
    "# print classification report\n",
    "target_names = [\"Fail = no\", \"Fail = yes\"]\n",
    "print(classification_report(target_test, target_predicted_dt, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Cross Validation of Decision Tree\n",
    "I cross validated with 10 repeats. You can see the OOB score for each repeat and the mean. The mean is .92, which is quite close to the orginal model. I am not going to worry about over fitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross Validation Score for each K [ 0.91694352  0.90697674  0.93355482  0.93355482  0.91362126  0.93979933\n",
      "  0.91638796  0.92976589  0.92976589  0.9264214 ]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.92467916310181231"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#verify DT with Cross Validation\n",
    "scores = cross_val_score(clf, features_train, target_train, cv=10)\n",
    "print(\"Cross Validation Score for each K\",scores)\n",
    "scores.mean()                             "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Visual of Confusion Matrix for Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1634   74]\n",
      " [  78  214]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARMAAAD0CAYAAAC4n8I2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGDhJREFUeJzt3Xm0HGWZx/Hvj4QtEEAIKgQwiAEHcwQREfGIuAyLIqhn\nVBDcxwgqLriBoqDjwlFHRXHUIBgUxeDCGQbQgMwg4mGPbBEMICKRsAtiQEjufeaP923pXO7tru6u\nvnW7+vfJqZPuquqqtzrpp9+t61FEYGbWq7WqLoCZ1YODiZmVwsHEzErhYGJmpXAwMbNSOJiYWSkc\nTMysFA4mFZO0vqT/kfSgpJ/0cJxDJJ1XZtmqIulFkv5QdTmsM/KktWIkvRE4Engm8BBwNfC5iLi4\nx+O+CTgC2CMiVvdc0ClOUgBzI+Lmqsti5ZpedQEGgaQjgaOAw4DFwGPAPsABQE/BBHgasGwYAkkR\nkqYP03uxz0s2iPvuHym071XXPro4Ivbtc5G6FxFeWizAxsDfgde12Gdd4GvAHXn5GrBu3rYXsBz4\nEHA3sAJ4W972aVJgWpXP8Q7gOOC0pmPPAQKYnp+/FfgjqXZ0K3BI0/qLm163B3AF8GD+e4+mbRcC\n/wH8Nh/nPGDWBNfWKP9Hm8r/auAVwDLgfuDjTfvvBlwCPJD3PRFYJ2+7KF/Lyny9b2g6/seAO4Ef\nNNbl12yXz7FLfr4lcA+wV9X/N8pYdnn2urFqxXaFFuDKqsvbanGfSXsvANYDzmyxzyeA3YGdgZ1I\nH6hjmrY/lRSUZpMCxjclPSkijgU+DyyKiA0j4uRWBZG0AfB1YL+ImEkKGFePs9+mwDl5382ArwDn\nSNqsabc3Am8DngysA3y4xamfSnoPZgOfAk4CDgWeC7wI+KSkbfO+I8AHgVmk9+5lwLsBImLPvM9O\n+XoXNR1/U1ItbX7ziSPiFlKgOU3SDOB7wKkRcWGL8g6QYCRGCy1FSDpF0t2Srh+z/ghJN0paKumL\nTeuPlnSzpD9I2qdp/XMlXZe3fV2S2p3bwaS9zYB7o3XV+xDgMxFxd0TcQ6pxvKlp+6q8fVVEnEv6\nVt6hy/KMAvMkrR8RKyJi6Tj7vBK4KSJ+EBGrI+J04EbgVU37fC8ilkXEI8AZpEA4kVWk/qFVwI9J\ngeKEiHgon//3pCBKRFwVEZfm8/4J+A7w4gLXdGxEPJrLs4aIOAm4GbgM2IIUvGshgFGi0FLQQmCN\nppCklwAHkoL4s4Av5/U7AgcBz8qv+S9J0/LLvgW8E5ibl7bNKweT9u4DZklq1b+0JXBb0/Pb8rp/\nHmNMMHoY2LDTgkTESlLT4DBghaRzJD2zQHkaZZrd9PzODspzX0Q0GvaND/tdTdsfabxe0vaSzpZ0\np6S/kWpes1ocG+CeiPhHm31OAuYB34iIR9vsO1BGC/4pIiIuIjULmx0OHN943yLi7rz+QODHOYjf\nSgrYu0naAtgofykE8H1S07YlB5P2LgEepfWbeQepit6wTV7XjZXAjKbnT23eGBGLI+JfSd/QN5I+\nZO3K0yjTX7osUye+RSrX3IjYCPg40K6K3PJrV9KGpH6ok4HjcjOuFoJgJIotpC+1K5uW+e2On20P\nvEjSZZJ+Lel5ef1s4Pam/ZbndbPz47HrW3IwaSMiHiT1E3xT0qslzZC0tqRjJN0v6WbSt/4xkjaX\nNCvvf1qXp7wa2FPSNpI2Bo5ubJD0FEkH5r6TR0nNpfG+ss4Ftpf0RknTJb0B2BE4u8sydWIm8Dfg\n77nWdPiY7XcBTy9wnCc1tf1PIHU+/jupL+jbZRa4ah00c+6NiF2blgUFTzGd1Ce1O/AR4IwifSCd\ncjApICL+kzTH5BjSSMLtpA/5fNKHdBZphOVa4DpgCfDZLs91PrAoH+sq1gwAa+Vy3EGqyr6YJ35Y\niYj7gP1JI0j3kUZi9o+Ie7spU4c+TOrcfYhUa1o0ZvtxwKmSHpD0+hbHWUlqp8/Mfzeu80hgF0mH\nlFnoqgQwQhRaerAc+Hkkl5O+gGaRaqpbN+23VV73l/x47PqWPGmtC5JeABwXEfvk50cDRMQXKi1Y\nzUiaA5wdEfMqLkrf7LTTOrH43HZdSskWW624KiJ2bbff2PdN0mHAlhHxKUnbAxeQmr07Aj8ijT5u\nmdfPjYgRSZcD7yN1ep9L6qs6t9V5PWmtO+O1NZ9fUVlswBXrWi1G0umkeTqzJC0HjgVOAU7JTcbH\ngLfkjtWlks4gjcatBt7T1NH+btLI0PrAL/LSkoOJWYWi9ybMmseLOHiCTYdOsP/ngM+Ns/5K0uhZ\nYQ4m3ZmorWnWmYCRmvQ0uAO2O1cAcyVtK2kd0sSfsyoukw2gNGmt2DLVOZh0IU9Aey/pR383AGdM\nMBPVupTb/pcAO0haLukdVZepP8RIwWWqczOnS7lnu2XvtnWvRdu/VgIYrUkzx8HErEIBPFaTBoKD\niVnFRmPqN2GKcDAxq1CaAetgYmY9CsRITZo59biKinTwq03r0jC8x6OhQstU52DSm9r/R58Cav0e\nN5o5Hho2sx6JkajHd/qUCiazNp0Wc7Zeu+piFLbN7OnsutN6AzVLYNm1M9rvNIWsxww20qYD9R7/\ng5U8Fo8WqkoEsIppbfcbBFMqmMzZem0uX7x1+x2ta/ts2epWr1aGy+KCwvtGuGZiZiUZHYD+kCIc\nTMwqlDpgXTMxs565mWNmJUi3IHAwMbMeBeKx8GiOmZVgtCbNnHpchdmAanTAFlmKmCjXcN72IUmR\nczs11jnXsFkdBGIkii0FLWScvMCStgb2Bv7ctM65hs3qZJS1Ci1FTJBrGOCrpGRszbOJS8017D4T\nswpF0PehYUkHAn+JiGvGtFZmA5c2PW/kFF5FF7mGHUzMKqVOZsDOknRl0/MF7fINS5pBSh6/d5cF\nLMzBxKxCATwWhT+G9xZJDzrGdsC2QKNWshWwRNJulJxr2H0mZhUKit0YqdubI0XEdRHx5IiYExFz\nSE2WXSLiTlKup4MkrStpW1JH6+URsQL4m6Td8yjOm4H/bncuBxOzipU8NFw431DO9dTINfxLnphr\n+LukTtlbcK5hs6kt5c0p7zu9Xb6hXDtpfu5cw2b1MBi3ZCzCwcSsQmXXTKrkYGJWMddMzKxnEWLV\naD0+hvW4CrMBle5n4pqJmfXMd1ozsxKkDljXTMysBL6htJn1rDGdvg4cTMwq5htKm1nPImDVqIOJ\nmfUoNXMcTMysBJ4Ba2Y989CwmZXEzRwzK4mn05tZz9Ld6R1MzKxHgVg96lzDZlYCN3PMrGd1Gs2p\nRzey2QAbjbUKLUWMl7hc0pck3SjpWklnStqkaZsTl5vVQsGcOR3UXhbyxCTj5wPzIuLZwDLgaHDi\ncrNaadxprchS6HjjJC6PiPMiYnV+eimPZ+tz4nKzOumg1tFxruFxvB1YlB87cblZXQSwuvivhrvJ\nNfxPkj4BrAZ+2O0xWulrM0fSvrlj52ZJR/XzXGaDqN+5hhskvRXYHzgkN11gUBKX546cbwL7ATsC\nB+cOHzNrUmafyXgk7Qt8FDggIh5u2lRq4vJ+NnN2A26OiD8CSPoxqcPn9308p9lgiXLnmeTE5XuR\n+leWA8eSRm/WBc7PI7yXRsRhEbFUUiNx+WqemLh8IbA+KWl5pYnLZwO3Nz1fDjx/7E6S5gPzAbaZ\n7S4cGy5lT1qbIHH5yS32Ly1xeeVDwxGxICJ2jYhdN9+sHr9RMOvEZPSZTIZ+VgUm6twxsywQIzW5\nB2w/r+IKYK6kbSWtQ5ppd1Yfz2c2kPrdATtZ+lYziYjVkt4LLAamAadExNJ+nc9sEEXJHbBV6muP\nZ0ScC5zbz3OYDbpwMDGz3g1G52oRDiZmFXPNxMx6VqebIzmYmFXJN5Q2szIEbuaYWSncAWtmJfnn\nDQEGnIOJWcXczDGznkU4mJhZSdxnYmalGB11MDGzHgVyM8fMylGTwRwHE7NK1agDth63eDIbZFFw\nKWCCXMObSjpf0k357yc1bet/rmFJG7Vail2ambUToUJLQQt5Yl7go4ALImIucEF+Xnqu4VbNnKWk\neNh8FY3nAWzT7uBm1l6ZM2Aj4iJJc8asPpCU/gLgVOBC4GM05RoGbpXUyDX8J3KuYQBJjVzDLdNd\nTBhMImLribaZWTkiIPp/Q+mn5MRaAHcCT8mPS801XOgqJB0k6eP58VaSnlvkdWbWXpoF234hJy5v\nWuZ3fq7ooAemM21HcySdCKwN7Al8HngY+DbwvH4UyGzoFP9od5u4/C5JW0TECklbAHfn9ZOea3iP\niHgX8A+AiLgfWKfA68ysrWKdrz0OH58FvCU/fguP5w2e9FzDqyStRY6fkjYDRju6FDObWImNjgly\nDR8PnCHpHcBtwOsBqsg1/E3gZ8Dmkj6dC/LpohdnZi2UPGltglzDAC+bYP/Scg23DSYR8X1JVwEv\nz6teFxHXt3qNmXWgJvPpi06nn0YaLgo8a9asXMMynV7SJ4DTgS1Jvbo/knR0vwtmNjRKnE5fpSI1\nkzcDz4mIhwEkfQ74HfCFfhbMbCgEtamZFAkmK8bsNz2vM7MS1P6G0pK+Soqb9wNLJS3Oz/cGrpic\n4pkNgboHE6AxYrMUOKdp/aXj7Gtm3ap7MyciTp7MgpgNpQDVZApokd/mbEea1LIjsF5jfURs38dy\nmQ0J1aZmUmTOyELge6T7mOwHnAEs6mOZzIZLTYaGiwSTGRGxGCAibomIY0hBxczKUJNgUmRo+NH8\nQ79bJB1G+inyzP4Wy2yIDECgKKJIMPkgsAHwPlLfycbA2/tZKLOhMUyT1iLisvzwIeBN/S2O2fBR\n3Wsmks6kRQUsIl7blxKZDZu6BxPgxEkrRbbs2hnss+XOk33aoTL9ab5PeL/pjrU727/uwSQiLpjM\ngpgNrWHpMzGzPhqQYd8iHEzMqjZswUTSujnzl5mVqC59JkXutLabpOuAm/LznSR9o+8lMxsW5SYu\n/6CkpZKul3S6pPW6SVzejSLT6b8O7A/cBxAR1wAv6eWkZpYo/2q4yNL2WNJs0uTSXSNiHunezQfR\nXeLyjhUJJmtFxG1j1o2Mu6eZdS5UbClmOrC+pOnADOAOUoLyU/P2U0lJyKEpcXlE3ArcDOzW7WUU\nCSa3S9oNCEnTJH0AWNbtCc1sjOLNnJa5hiPiL8CXgT+Tbq36YEScR+vE5bc3HaJQgvKJFOmAPZzU\n1NkGuAv4VV5nZiXooAO2Za7h3BdyILAt8ADwE0mHNu8TESH1p8u3yG9z7ia1q8ysH8r7aL8cuDUi\n7gGQ9HNgDzpPXN6VIndaO4lxLjci5o+zu5l1IkodGv4zsLukGcAjpJSgVwIrSQnLj+eJict/JOkr\npLxYc4HLuz15kWbOr5oerwe8hjXbWWbWi5KCSURcJumnwBJSIvLfAQuADek8cXnHijRz1rhFo6Qf\nABd3e0IzW1OZN5SOiGOBY8esfpQOE5d3o5u8wdvyeG+wmRlQrM/krzxeEVuLlJTrqH4Wymyo1GQ6\nfctgIknATjzewzsaUZdkhmZTQLkdsJVq2czJgePciBjJS00u22wKqcnd6Yv0mVwt6Tl9L4nZsKpJ\nMGl1D9jpEbEaeA5whaRbSOPVIlVadpmkMprVlqhPM6dVn8nlwC7AAZNUFrPhMyS5hgUpi98klcVs\nOA1BzWRzSUdOtDEivtKH8pgNnyEIJtNI03DrcetssylqGPpMVkTEZyatJGbDagiCiWskZv02IMO+\nRbQKJuP+MMjMylX70ZyIuH8yC2I2rIahz8TMJoODiZn1bEj6TMysz0R9RjocTMyq5pqJmZXBHbBm\nVo6aDA13cw9YMytLvtNakaUISZtI+qmkGyXdIOkFUylxuZn1U7k3RzoB+GVEPJN0y9UbmEKJy82s\nj8qqmUjaGNgTOBkgIh6LiAeYQonLzayfSkpcTkpDcw/wPUm/k/RdSRswhRKXm1kflZW4nPR53gU4\nImf3O4ExaWn6mbjcNROzKhWtlRT7+C8HlkfEZfn5T0nB5a6csJx+Ji53MDGrkEi/Gi6ytBMRdwK3\nS9ohr3oZKY/wWaSE5fDExOUHSVpX0rZMQuJyM+unchsdRwA/lLQO8EfgbaRKQ/WJy7sl6RRgf+Du\niJjXr/OYDTqVmNsuIq4GxutXmZKJy4taSBq7NrOJlNtnUqm+1Uwi4iJJc/p1fLO68G9zSpLHyucD\nrMeMiktjVgEHk3JExAJgAcBG2rQmb6tZca6ZmFnvhiQ9qJlNhprUTPo2miPpdOASYAdJy/MYt5k1\nEeXegqBK/RzNObhfxzarlRLnmVTJzRyzig1CraMIBxOzKg3IhLQiHEzMKubRHDMrhYOJmfUucAes\nmZXDHbBmVg4HEzPrVWPSWh04mJhVKcJ9JmZWDo/mmFkp6tLM8d3pzaoUwGgUWwqSNC0n4To7P3eu\nYbOhUP49YN9PyjHc4FzDZsOgzFsQSNoKeCXw3abVzjVsNhQaIzrtlmK+BnwUaO7WnZRcww4mZhXr\noGbSMnG5pEaeqqsmOldE9O13yh7NMauQAlS8c7Vd4vIXAgdIegWwHrCRpNPIuYYjYoVzDZvV2WjB\npY2IODoitoqIOaSO1f+NiENxrmGz4VBmetAJHM8g5xo2swL61IMRERcCF+bH9zEJuYYdTMwq5d/m\nmFlJ6jKd3sHErGqumZhZzwI04mBiZmWoRyxxMDGr2iQMDU8KBxOzqjmYmFnPgkKzWweBg4lZhUS4\nmWNmJXEwMbOeBeChYTMrg5s5ZlYOBxMz651/6GdmZQgcTMysJJ5nYmZlcAesmfUugJF6VE0cTMwq\n5Q7YvniIv977q/jpbVWXowOzgHurLkRH/lR1ATo2eO8xPK2jvR1MyhcRm1ddhk5IurJNHhPr0VC8\nxyUFE0lbA98nZewLYEFEnCBpU2ARMIf0dfL6iPhrfs3RwDuAEeB9EbG42/M7b45ZlQIYjWJLe6uB\nD0XEjsDuwHtycnInLjerv4AYLba0O1LEiohYkh8/BNxAyh08KYnLp1QzZwAtqLoAQ6De73Fnozmz\nJF3Z9HxBRIz7/kiaAzwHuIzWicsvbXqZE5dXZaJ/yAZJI5KulnS9pJ9ImtHtuSTtJens/PgASUe1\n2HcTSe/u4hzHSfpw0fVj9lko6d86ONccSde326/de1wLEcWWnGu4aZkokGwI/Az4QET8bc1T9S9x\nuYNJfz0SETtHxDzgMeCw5o1KOv43iIizIuL4FrtsAnQcTKwixYNJW5LWJgWSH0bEz/Pqu3LCcpy4\nvB5+AzwjfyP/QdL3geuBrSXtLekSSUtyDWZDAEn7SrpR0hLgtY0DSXqrpBPz46dIOlPSNXnZg5Rb\ndrtcK/pS3u8jkq6QdK2kTzcd6xOSlkm6GNih3UVIemc+zjWSfjamtvVySVfm4+2f958m6UtN535X\nr29kvRQMJAWCiSQBJwM3RMRXmjY5cXldSJoO7Af8Mq+aC7wlIi6VNAs4Bnh5RKyU9DHgSElfBE4C\nXkrqGFs0weG/Dvw6Il6Te+I3JPXWz4uInfP5987n3A0QcJakPYGVpN78nUn/F5YAV7W5nJ9HxEn5\nuJ8lDSt+I2+bk8+xHfB/kp4BvBl4MCKeJ2ld4LeSzqM2CR56FMBoaTNgXwi8CbhO0tV53cdx4vJa\nWL/pH/U3pG+NLYHbIqLR8bU7sCPpQwawDnAJ8Ezg1oi4CUDSacD8cc7xUtIHlvwf4UFJTxqzz955\n+V1+viEpuMwEzoyIh/M5zipwTfNyENkkH6d5XsIZETEK3CTpj/ka9gae3dSfsnE+97IC5xoOJc0z\niYiLSV8W43Hi8gH3SKN20JADxsrmVcD5EXHwmP3WeF2PBHwhIr4z5hwf6OJYC4FXR8Q1kt4K7NW0\nbeynIvK5jxg7GSqPNhjUZgas+0yqdynwwtwkQNIGkrYHbgTmSNou73fwBK+/ADg8v3aapI2Bh0i1\njobFwNub+mJmS3oycBHwaknrS5oJvKpAeWcCK3JH3yFjtr1O0lq5zE8H/pDPfXjeH0nbS9qgwHmG\nQwQxMlJomepcM6lYRNyTv+FPz30KAMdExDJJ84FzJD1MaibNHOcQ7wcW5PbwCHB4RFwi6bd56PUX\nEfERSf8CXJJrRn8HDo2IJZIWAdeQevivKFDkT5LmLtyT/24u059JHXgbAYdFxD8kfZfUl7IkdxDe\nw+OTpgyKzm6d8hQ1qWKZDaKNp28eL5h5YKF9Fz9w8lVT+XdKrpmYVSmizNGcSjmYmFWtJq0DBxOz\nioVrJmbWO99pzczKEMAADPsW4WBiVqEAoiZDww4mZlWKKHTjo0HgYGJWsbrUTDxpzaxCkn5JugN/\nEfdGxL79LE8vHEzMrBT+oZ+ZlcLBxMxK4WBiZqVwMDGzUjiYmFkpHEzMrBQOJmZWCgcTMyuFg4mZ\nleL/AYrpg9FszmmTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113fe08d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# display confusion matrix\n",
    "cm = confusion_matrix(target_test, target_predicted_dt)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Random Forest\n",
    "Using the same data, I built a random forest with 500 bootstrapped trees. Notice I parallelized this to 4 cores as big random forest can be computationally expensive. \n",
    "\n",
    "My overall results went up by 3% over the decision tree. Also, my minory target precision, but the recall decresed.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9495\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      " Churn = no       0.95      0.99      0.97      1708\n",
      "Churn = yes       0.96      0.68      0.80       292\n",
      "\n",
      "avg / total       0.95      0.95      0.95      2000\n",
      "\n",
      "[[1699    9]\n",
      " [  92  200]]\n"
     ]
    }
   ],
   "source": [
    "# train random forest model\n",
    "#paralleized to 4 cores \n",
    "rf = RandomForestClassifier(n_estimators= 500, n_jobs=-1,oob_score=True)\n",
    "rf.fit(features_train, target_train)\n",
    "# test random forest model\n",
    "target_predicted_rf = rf.predict(features_test)\n",
    "print(accuracy_score(target_test, target_predicted_rf))\n",
    "target_names = [\"Churn = no\", \"Churn = yes\"]\n",
    "print(classification_report(target_test, target_predicted_rf, target_names=target_names))\n",
    "print(confusion_matrix(target_test, target_predicted_rf))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Cross Validation of Random Forest\n",
    "I cross validated with 10 repeats. You can see the OOB score for each repeat and the mean. The mean is .949, which is quite close to the orginal model. I am not going to worry about over fitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross Validation Score for each K [ 0.94684385  0.94352159  0.95016611  0.95681063  0.94019934  0.94983278\n",
      "  0.95317726  0.94648829  0.94983278  0.95652174]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9493394371048568"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#verify RF with cross validation\n",
    "scores_rf = cross_val_score(rf, features_train, target_train, cv=10, n_jobs=-1)\n",
    "print(\"Cross Validation Score for each K\",scores_rf)\n",
    "scores_rf.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Visual of Confusion Matrix for Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1699    9]\n",
      " [  92  200]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARMAAAD0CAYAAAC4n8I2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGC1JREFUeJzt3XmUXGWZx/HvL4kkhAQUgyIhGMQAIscg2yAeHRgdNhEc\nz6ggiwsjAyqj4jKAOKAjIzM6qLiziYoguDAygAZkxgUPW9iJQAAVDYYlMAICkqT7mT/et02l6a66\nVXWrb9et38dzT6ruvXXftxr76Xe791FEYGbWrSlVV8DM6sHBxMxK4WBiZqVwMDGzUjiYmFkpHEzM\nrBQOJmZWCgeTiklaV9J/S3pU0ne7uM5Bki4rs25VkfQqSXdWXQ9rj7xorRhJbwWOBrYGHgduAk6K\niCu7vO4hwFHArhGxuuuKTnKSAlgQEXdXXRcr17SqK9APJB0NHAMcASwCVgJ7AvsBXQUT4IXA0kEI\nJEVImjZIP4s9d18vHn5kqNC519/y9KKI2KvHVepcRHhrsgEbAH8C3tTknOnA54A/5O1zwPR8bDdg\nGfBB4EFgOfCOfOzjpMC0KpdxGHAicE7DtecDAUzL798O/JrUOvoNcFDD/isbPrcrcB3waP5314Zj\nPwX+Ffhlvs5lwJxxvttI/T/SUP83APsAS4FHgOMazt8ZuAr4Yz73i8A6+djP83d5In/ftzRc/5+B\n+4FvjezLn9kil7F9fr8J8BCwW9X/3yhj2/5l02PV8i0KbcDiquvbbPOYSWuvAGYAFzY556PALsB2\nwELSL9TxDcc3JgWluaSA8SVJz4mIE4B/A86PiFkRcWazikhaDzgV2DsiZpMCxk1jnLchcEk+97nA\nKcAlkp7bcNpbgXcAzwPWAT7UpOiNST+DucC/AKcDBwM7AK8CPiZp83zuEPABYA7pZ/ca4N0AEfHq\nfM7C/H3Pb7j+hqRW2uGNBUfEPaRAc46kmcDXgW9ExE+b1LePBEMxXGib7BxMWnsusCKaN70PAj4R\nEQ9GxEOkFschDcdX5eOrIuJS0l/lrTqszzCwraR1I2J5RCwZ45zXAXdFxLciYnVEnAfcAby+4Zyv\nR8TSiHgKuIAUCMezijQ+tAr4DilQfD4iHs/l/4oURImI6yPi6lzub4GvAX9d4DudEBFP5/qsJSJO\nB+4GrgFeQAretRDAMFFom+wcTFp7GJgjqdn40ibAvQ3v7837/nKNUcHoSWBWuxWJiCdIXYMjgOWS\nLpG0dYH6jNRpbsP7+9uoz8MRMdKxH/llf6Dh+FMjn5e0paSLJd0v6TFSy2tOk2sDPBQRf25xzunA\ntsAXIuLpFuf2leGC/5vsHExauwp4mjROMJ4/kJroIzbL+zrxBDCz4f3GjQcjYlFE/C3pL/QdpF+y\nVvUZqdN9HdapHV8h1WtBRKwPHAeoxWea/tmVNIs0DnUmcGLuxtVCEAxFsW2yczBpISIeJY0TfEnS\nGyTNlPQsScdLekTS3aS/+sdL2kjSnHz+OR0WeRPwakmbSdoAOHbkgKTnS9o/j508TeoujfUn61Jg\nS0lvlTRN0luAbYCLO6xTO2YDjwF/yq2mI0cdfwB4UYHrPEfSg5JuAz5PGnz8B9JY0FfLrHDV3M0Z\nIBHxn6Q1JseTZhJ+T/olP5z0SzqHNMNyC3ArcAPwyQ7Luhw4P1/retYOAFNyPf5AmuH4a575y0pE\nPAzsS5pBepg0E7NvRKzopE5t+hBpcPdxUqvp/FHHTwS+IemPkt7c5DpPAHuRgtNerPmeRwPbSzqo\nzEpXJYAhotA22XnRWgckvQI4MSL2zO+PBYiIT1VasZqRNB+4OCK2rbgqPbNw4Tqx6NJWQ0rJCzZd\nfn1E7NjjKnXMLZPOzCW1TkYsY+3BTbPChgtuk52DiVmFomAXp2g3R9JZDWNNjfuPknSHpCWS/qNh\n/7GS7pZ0p6Q9G/bvIOnWfOxUSa0G0R1MOnQfMK/h/aZMzEyJ1U3AUMGtoLNJY0x/IWl3YH/SYsGX\nAp/J+7cBDgBemj/zZUlT88e+ArwLWJC3lsv4HUw6cx2wQNLmktYh/Qe5qOI6WR9Ki9bK6+ZExM9J\ng/ONjgROHlmfExEP5v37A9/JiwV/Q1oYuLOkFwDr58WHAXyT5ksjAAeTjuQFaO8l3fR3O3DBOCtR\nrUOSziOt8dlK0jJJh1Vdp94QQwU30uLJxQ3b4a2unm0JvErSNZJ+JmmnvH+8sb+5+fXo/U35ruEO\n5WXxl1Zdj7qKiAOrrsNECGC4eBdmRYezOdNI9z7tAuwEXCCpyFqftgsxs4oEsLL3HYRlwA9yl+Va\nScOktVHjjf3dl1+P3t+UuzlmFRsOFdq68F/A7pDunSLdJb6CNM53gKTp+a7vBcC1EbEceEzSLnkW\n51Dgh60KccvErEJpBWxXgWIteaxpN9L4yjLgBOAs4Kw8XbwSeFtupSyRdAHpru/VwHsabuh8N2lm\naF3gR3lrXrZXwJpV5yUvmx7fvPgFhc7d+YX3egVsXbUxmm4dGoSf8QR0cyaEg0l3av9/9Emg1j/j\nkW5OwanhSc1jJmaVEkNRj7/pkyqYzNlwasyf96yqq1HYZnOnsePCGX016LT0lpmtT5pEZjCT9bVh\nX/2M/8wTrIynCzUlAljF1Jbn9YNJFUzmz3sW1y6a1/pE69iemzR71KuV4Zq4ovC5EW6ZmFlJhvtg\nPKQIBxOzCqUBWLdMzKxr7uaYWQnSIwgcTMysS4FYGZ7NMbMSDLubY2bd8gCsmZUiEEN9cN9NEQ4m\nZhXzAKyZdS0CTw2bWRnkFbBm1r0AVkY9fg3r0b4y61NBsQcjFX040ngZ/fKxD0oKSXMa9jmjn1ld\nDDGl0FbQ2YyRfU/SPGAP4HcN+5zRz6wuUt6cKYW2QtcbO6MfwGeBj+QiR5Sa0a8enTWzvtX7RzJK\n2h+4LyJuHtVbmQtc3fB+JHPfKpzRz6y/jLRMCpojaXHD+9Mi4rRmH5A0EziO1MXpKQcTs4q10TLp\nJD3oFsDmwEirZFPgBkk7U3JGPwcTswpFiFXDvfs1jIhbgeeNvJf0W2DHiFgh6SLgXEmnAJuwJqPf\nkKTHJO0CXEPK6PeFVmV5ANasQul5Jiq0FZEz+l0FbCVpmaTDxi07YgkwktHvxzwzo98ZpEHZeyiQ\n0c8tE7NKlfuktYg4sMXx+aPenwScNMZ5i4Ft2ynbwcSsQmkA1svpzawEfp6JmXVtZDl9HTiYmFXM\nzzMxs65FwKphBxMz61Lq5jiYmFkJen1vzkRxMDGrkKeGzawk7uaYWUn8DFgz61p6Or2DiZl1KRCr\nh51r2MxK4G6OmXXNszlmVhrP5phZ99rIiTPZOZiYVWjkSWt14GBiVrG6tEzq0Vkz61MBrB6eUmgr\nYqz0oJI+LekOSbdIulDSsxuO9Ud6UEl75UreLemYXpZl1o/KzjXM2OlBLwe2jYiXAUuBY6GP0oPm\nSn0J2BvYBjgwV97MGpT5dPqx0oNGxGURsTq/vZo1OXFKTQ/ay5bJzsDdEfHriFgJfCdX3sxGBO20\nTOZIWtywHd5Bie9kTdqKucDvG46NpAGdyyRLDzpWRf9q9En5B3I4wGZzPR5sg6XNRWudZPT7C0kf\nBVYD3+70Gs1U/tubc6WeBrDjwhnR4nSz2pmI2RxJbwf2BV6Tuy5QcnrQXnZzxquomWWBGBqeUmjr\nlKS9gI8A+0XEkw2HLgIOkDRd0uasSQ+6HHhM0i55FudQ4Ietyully+Q6YEGu5H2kUeO39rA8s75U\n5qK1nB50N9L4yjLgBNLszXTg8jzDe3VEHBERSySNpAddzTPTg54NrEsaY6kuPWhErJb0XmARMBU4\nK+c2NbMsotxuzjjpQc9scn5/pAeNiEuBS3tZhlm/i5qsgK18ANZssPlGPzMriVsmZtY1PxzJzMrh\nB0qbWRkCd3PMrBQegDWzkkRNbiJxMDGrmLs5Zta1CAcTMyuJx0zMrBTDww4mZtalQO7mmFk5ajKZ\n42BiVikPwJpZaWrSNBk3mEhav9kHI+Kx8qtjNnjq0jJp9mDJJcBt+d8lo97f1uRzZtaGtNak9VbE\nOBn9NpR0uaS78r/PaTjW+4x+ETEvIjbL/84b9X6zYl/NzJqJgBieUmgr6GyemX3vGOCKiFgAXJHf\nV5PRT9IBko7LrzeVtEORz5lZa2W2TMbK6EdKfveN/PobrMnON7EZ/SR9EdgdOCTvehL4aqvPmVlB\nUXDrPKPf83P6CoD7gefn1xOe0W/XiNhe0o0AEfGIpHUKfM7MWmpr0VpXGf0AIiIk9WT+qEg3Z5Wk\nKeTYKOm5wHAvKmM2kIq3TDr1QO66kP99MO+f8Ix+XwK+D2wk6ePAlcC/F/icmbWSF60V2bpwEfC2\n/PptrMnON7EZ/SLim5KuB16bd70pIjw1bFaWEjsd42T0Oxm4QNJhwL3AmwGqyug3FVhF+tq9zE9s\nNnh6n9EP4DXjnF9aRr8iszkfBc4DNiH1nc6VdGw7hZhZE70fM5kQRVomhwIvH8meLukk4EbgU72s\nmNlACEptmVSpSDBZPuq8aXmfmZWg9g+UlvRZUtx8BFgiaVF+vwdw3cRUz2wA1D2YsOZmviXAJQ37\nr+5ddcwGUN27ORFx5kRWxGwgBagmS0BbjplI2oI0dbQNMGNkf0Rs2cN6mQ0I1aZlUmTNyNnA1wEB\newMXAOf3sE5mg6UmU8NFgsnMiFgEEBH3RMTxpKBiZmWoSTApMjX8dL7R7x5JR5Bu+Jnd22qZDZA+\nCBRFFAkmHwDWA/6JNHayAfDOXlbKbGAM0qK1iLgmv3ycNQ9IMrOS9ObpIhOv2aK1C2nSAIuIN/ak\nRmaDpu7BBPjihNUiu2vJbPZ56e4TXexAmbrlhlVXofb02yvbO7/uwSQirpjIipgNrEEZMzGzHuqT\nad8iHEzMqlaTYFL4qWmSpveyImaDSlFsK3Qt6QOSlki6TdJ5kmZ0ktGvE0WetLazpFuBu/L7hZK+\n0E2hZtagpBWwkuaS1oPtGBHbkh63egCdZfRrW5GWyanAvsDDABFxMykpl5l1Sfmu4SJbQdOAdSVN\nA2YCf6DNjH6dfpciwWRKRNw7at/QmGeaWftCxbZWl4m4D/gM8DvS0xAfjYjLaD+jX0eKBJPfS9oZ\nCElTJb0fWNppgWY2SknpQfNYyP7A5qQHwK8n6eC1ikq5g3sy5FtkNudIUldnM+AB4Cd5n5mVoI1F\na63Sg74W+E1EPAQg6QfAruSMfhGxvGBGv44UuTfnQdIgjZn1QnnthN8Bu0iaCTxFypWzGHiClMnv\nZJ6Z0e9cSaeQWjILgGs7LbzIk9ZOZ4yvGxFFM7Cb2XjamPZteamIayR9D7iBlKHvRuA0YBbtZ/Rr\nW5Fuzk8aXs8A/o61B23MrBsljmBExAmklKCNnqbNjH6dKNLNWesRjZK+RUpebmYlqMsDpTvJG7w5\na6aWzMyAYmMm/8eahtgUUlKuY3pZKbOBUpN7c5oGE0kCFrJmumg4z1ObWRlKHICtWtNuTg4cl0bE\nUN5q8rXNJpGaPJ2+yJjJTZJe3vOamA2qmgSTZs+AnRYRq4GXA9dJuoe0+EWkRsv2E1RHs9oS9enm\nNBszuRbYHthvgupiNngGJNewIGXxm6C6mA2mAWiZbCTp6PEORsQpPaiP2eAZgGAylbSmvx6Pzjab\npAZhzGR5RHxiwmpiNqgGIJi4RWLWa30y7VtEs2Ay5l2GZlau2s/mRMQjE1kRs0E1CGMmZjYRHEzM\nrGsDMmZiZj0m6jPT0cnDkcysTCXe6Cfp2ZK+J+kOSbdLesWkSQ9qZr1VZq5h4PPAjyNia9KziG5n\nEqUHNbNeGi64tSBpA+DVwJkAEbEyIv7IJEoPama9UrBVogIZ/UjPZ34I+LqkGyWdIWk9Jig9qAdg\nzapWXka/aaTHhhyVc+h8nlHPa46IkHqzssUtE7OKlThmsgxYFhHX5PffIwWXB3JaUHqZHtTBxKxq\nJc3mRMT9wO8lbZV3vYaUre8iUlpQeGZ60AMkTZe0Ob1OD2pmvVVyp+Mo4NuS1gF+DbyD1GiYFOlB\nzaxXSl4BGxE3AWONq1SfHtTMekcMwF3DZjZBanJvTs8GYCWdJelBSbf1qgyzOlBEoW2y6+Vsztmk\nJbpmNp6iMzmTP5b0rpsTET+XNL9X1zerCz8cqSR5SfDhADOmzKq4NmYVqEkwqXzRWkScFhE7RsSO\n62hG1dUxm3Al3zVcmcpbJmYDbUDSg5rZROiDVkcRvZwaPg+4CthK0rK8lNfMGgh3c1qKiAN7dW2z\nWumDNSRFuJtjVrF+aHUU4WBiVqU+WZBWhIOJWcU8m2NmpXAwMbPuBR6ANbNy1GUAtvLl9GYDr+S7\nhiVNzakuLs7vndHPrO56tGjtfaRMfiOc0c+s9iKKbwVI2hR4HXBGw25n9DMbBBouthX0OeAjrJ1Q\ndEIy+jmYmFWsrPSgkvYFHoyI68crKyJ6tkzOszlmVQpguPDvdqv0oK8E9pO0DzADWF/SOeSMfhGx\n3Bn9zOqsvIx+x0bEphExnzSw+j8RcTDO6Gc2GCZgncnJOKOf2QDowQrYiPgp8NP8+mGc0c+s/uqy\nAtbBxKxCClDxAdhJzcHErGq+a9jMytAPqT+LcDAxq5KftGZm5Sh+381k52BiVjHP5phZOdwyMbOu\nBWjIwcTMylCPWOJgYlY1Tw2bWTkcTMysa4FXwJpZ90S4m2NmJXEwMbOuBeCpYTMrg7s5ZlaOmgQT\nP1DarFLlJeGSNE/S/0r6laQlkt6X9zs9qFntBWVm9FsNfDAitgF2Ad6TU4A6PajZQBguuLUQEcsj\n4ob8+nFSvuG5TFB6UI+ZmFWsjQHYOZIWN7w/LSJOG/Oa0nzg5cA1NE8PenXDx7pKD+pgYlalAIYK\nL4FtldEPAEmzgO8D74+IxyStKS4ipN48QcXdHLNKlTcACyDpWaRA8u2I+EHe/UBOC0ov04NOqpbJ\nY0MrVix65PR7q65HG+YAK6quRFseqboCbeu/nzG8sK2zS5oaVmqCnAncHhGnNBwaSQ96Ms9MD3qu\npFOATahTetCI2KjqOrRD0uIizU7r3ED8jMtbZ/JK4BDgVkk35X3H4fSgZgMggJKScEXElYDGOez0\noGb1FhD1eAaBg0l3xpyWs1LV+2fc3mzOpObZnC6MN8c/QtKQpJsk3Sbpu5JmdlqWpN0kXZxf7yfp\nmCbnPlvSuzso40RJHyq6f9Q5Z0v6+zbKmi/ptlbntfoZ10KJszlVcjDpraciYruI2BZYCRzReFBJ\n2/8NIuKiiDi5ySnPBtoOJlYRBxNr0y+AF+e/yHdK+iZwGzBP0h6SrpJ0Q27BzAKQtJekOyTdALxx\n5EKS3i7pi/n18yVdKOnmvO1KGr3fIreKPp3P+7Ck6yTdIunjDdf6qKSlkq4Etmr1JSS9K1/nZknf\nH9Xaeq2kxfl6++bzp0r6dEPZ/9jtD7Jeyl1nUiUHkwkgaRqwN3Br3rUA+HJEvBR4AjgeeG1EbA8s\nBo6WNAM4HXg9sAOw8TiXPxX4WUQsBLYHlpBu5Lont4o+LGmPXObOwHbADpJeLWkH0o1e2wH7ADsV\n+Do/iIidcnm3A4c1HJufy3gd8NX8HQ4DHo2InfL13yVp8wLlDIYAhoeLbZOcB2B7a92G+f5fkBYU\nbQLcGxEj90TsAmwD/DIve14HuArYGvhNRNwFIOkc4PAxyvgb4FCAvEbg0cZbzLM98nZjfj+LFFxm\nAxdGxJO5jIsKfKdtJX2S1JWaBSxqOHZBRAwDd0n6df4OewAvaxhP2SCXvbRAWYOhD1odRTiY9NZT\nEbFd444cMJ5o3AVcHhEHjjpvrc91ScCnIuJro8p4fwfXOht4Q0TcLOntwG4Nx0b/VkQu+6iIaAw6\nIzeiGdQmmLibU72rgVdKejGApPUkbQncAcyXtEU+78BxPn8FcGT+7FRJGwCPk1odIxYB72wYi5kr\n6XnAz4E3SFpX0mxSl6qV2cDyfA/IQaOOvUnSlFznFwF35rKPzOcjaUtJ6xUoZzBEEENDhbbJzi2T\nikXEQ/kv/HmSpufdx0fEUkmHA5dIepLUTZo9xiXeB5yWl0oPAUdGxFWSfpmnXn+Ux01eAlyVW0Z/\nAg6OiBsknQ/cTLr567oCVf4Y6bb2h/K/jXX6HenejvWBIyLiz5LOII2l3JDvHXmINc/TMChtBWzV\nFDVpYpn1ow2mbRSvmL1/oXMX/fHM6yfzfUpumZhVKaIvZmqKcDAxq1pNegcOJmYVC7dMzKx7/bG6\ntQgHE7MqBdAH075FOJiYVSiAqMnUsIOJWZXCD0cys5LUpWXiRWtmFZL0Y9IT+ItYERF79bI+3XAw\nMbNS+EY/MyuFg4mZlcLBxMxK4WBiZqVwMDGzUjiYmFkpHEzMrBQOJmZWCgcTMyvF/wOsye1w/TP9\nAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113fd24e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# display confusion matrix\n",
    "cm = confusion_matrix(target_test, target_predicted_rf)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Model Tuning\n",
    "You can tune any argument in these models. I did a grid search only on max_features (mtry in R). I parallelized the job to 4 cores for speed. You can see that max_features (mtry) of 5 had the best results. But frankly was very little difference from the other parameter results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.4734930000000013 seconds\n",
      "[mean: 0.88833, std: 0.00724, params: {'max_features': 2}, mean: 0.90200, std: 0.00611, params: {'max_features': 3}, mean: 0.91367, std: 0.00905, params: {'max_features': 4}, mean: 0.92300, std: 0.00782, params: {'max_features': 5}]\n"
     ]
    }
   ],
   "source": [
    "# use a full grid over all parameters\n",
    "param_grid = {\"max_features\": [2, 3, 4, 5]}\n",
    "start_time = time.clock()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# run grid search\n",
    "grid_search = GridSearchCV(rf, param_grid=param_grid,n_jobs=-1)\n",
    "\n",
    "grid_search.fit(features_train, target_train)\n",
    "\n",
    "print(time.clock() - start_time, \"seconds\")\n",
    "print(grid_search.grid_scores_)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#KNN\n",
    "I performed KNN on K=3 and K=5. For both K's the accurancy was 85% and 87% respectively and I still have problems with the minority class. KNN and Decision Tree perform about the same. I find this to be true frequently, which is why I use them as my base comparative models. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#More Details\n",
    "Now that we know our random forest was the best model of the three I ran, I will gather some other information. Below is a non-ordered list of feature importance. I only showed 20 for purposes of space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<zip at 0x114ef9c88>"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show importance of each feature in Random Forest\n",
    "zip(df.columns[1:20], rf.feature_importances_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#ROC curve for Random Forest\n",
    "Finally a ROC curve that shows the lift I get from the Random Forest model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC: 0.921\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGX2wPHvISQhIYEEQk0Q6QkdCQiighVUBAUVBBWx\nUITFugiorAIriLIioKDrgm0X0BVE+elSFEGkCUiVEopAQOmBVEg5vz9miAGSYYiZTCY5n+eZh9w7\nt5xcJvfMfe97zyuqijHGGJOXUt4OwBhjTNFmicIYY4xLliiMMca4ZInCGGOMS5YojDHGuGSJwhhj\njEuWKIwxxrhkicIUKyLyq4ikikiSiPwuIh+ISMgFy1wjIt+JSKKInBKRr0Sk4QXLlBORiSKy37mt\n3c7piDz2KyIyRES2iEiyiMSLyGci0sSTv68xhcEShSmO7lTVEKA50AIYfu4NEWkLLATmAdWBWsBG\n4EcRqe1cJgD4FmgEdALKAW2BY0DrPPb5FvAkMASoANQHvgDuuNzgRaT05a5jjCeJPZltihMR+RV4\nTFUXO6fHA41U9Q7n9A/AZlV94oL1vgGOqupDIvIY8HegjqomubHPesB2oK2qrsljme+BT1T1fef0\nw844r3VOKzAYeAooDfwPSFbV53JsYx6wVFX/ISLVgcnA9UAS8KaqTnLjEBlz2eyKwhRbIhIF3Abs\nck4HA9cAn+Wy+KfALc6fbwb+506ScLoJiM8rSVyGu4CrgYbATKCHiAiAiIQDtwKzRKQU8BWOK6FI\n5/6fEpGOf3L/xuTKEoUpjr4QkUTgAHAE+JtzfgUcn/nfclnnN+Dc/YeKeSyTl8tdPi9jVfWEqqYC\nPwAKXOd87x5gpaoeAloBlVR1lKqeVdU9wD+BngUQgzEXsURhiqO7VDUU6ABE80cCOAlkAdVyWaca\njnsQAMfzWCYvl7t8Xg6c+0EdbcKzgPuds3oB/3b+XBOoLiIJ517ACKBKAcRgzEUsUZhiS1WXAh8A\nbzink4GVwL25LH4fjhvYAIuBjiJS1s1dfQtEiUisi2WSgeAc01VzC/mC6ZnAPSJSE0eT1OfO+QeA\nvaoaluMVqqq3uxmvMZfFEoUp7iYCt4hIM+f0MKCPsytrqIiEi8gYHL2aXnEu8zGOk/HnIhItIqVE\npKKIjBCRi07GqhoHvAPMFJEOIhIgImVEpKeIDHMutgHoJiLBIlIXePRSgavqzziuct4HFqhqgvOt\nNUCiiDwvIkEi4icijUWkVX4OkDGXYonCFGuqehT4CBjpnF4OdAS64bivsA9HF9prnSd8VPUMjhva\n24FFwGkcJ+cIYHUeuxoCTAHeBhKA3cDdOG46A7wJnAUOAx/yRzPSpfzHGct/cvxOmUBnHN1/9/JH\nMinv5jaNuSzWPdYYY4xLdkVhjDHGJUsUxhhjXLJEYYwxxiVLFMYYY1zyueJjEREReuWVV3o7DGOM\n8Snr1q07pqqV8rOuzyWKK6+8krVr13o7DGOM8Skisi+/61rTkzHGGJcsURhjjHHJEoUxxhiXLFEY\nY4xxyRKFMcYYlyxRGGOMccljiUJEpovIERHZksf7IiKTRGSXiGwSkas8FYsxxpj88+QVxQdAJxfv\n3wbUc776AVM9GIsxxpQ4ZzIySUt3vP4Mjz1wp6rLRORKF4t0BT5yDvm4SkTCRKSaqhbE2MPGGFNs\npZzN4FBCGocSUkk6k3Heewkp6fy8/yTr959k99Fk0g5s4fRPX/yp/XnzyexIcowRDMQ7512UKESk\nH46rDq644opCCc4YYwra2YwsTiSf5VjSGY4lneF40lmOJzv+TUhJJyuP8YEURwI4lJDKoVOpJKSk\nu9xPeLA/LWuG07lpdb6f/R0/JsaT+ifi9okSHqr6HvAeQGxsrI20ZIwpNGczsjiYkEpSWkaeyyhK\nytnM7BP/saSzHL8gERxLOsPpPLYRULoU4cH++InkuY/QMv5EhgdxVc0wqocFERkWRPWwIELLlEb4\nY71AP2HxvFlUrXiWzre0YlD7sWRkjKZsWXeHgL+YNxPFQaBGjuko5zxjjClUSWcy2Hc8mf3HU9h3\nIoV9x1PYfyKZfcdTOJSQStZlfj0VgQrBAVQMCaBi2UAaVi9HREggFcsGUDEkkIohAUQ436sYEkBI\nYGnERZJw15YtW3hs4ECWL1/O/fffT+fOnQkICCAgIOBPbdebieJLYLCIzAKuBk7Z/QljjKedTktn\n7a8nWL3nBGv3neTXY8kcTz573jIVygZwRYVgWtYMp1uLSK6oWJawIH+X2w0K8HMkg5AAwoMD8Cv1\n50/87kpOTmb06NFMmDCB8uXLM2PGDPr06VNg2/dYohCRmUAHIEJE4oG/Af4AqjoN+Bq4HdgFpAB9\nPRWLMabkOpWSzppfT7B6z3FW7z3B1kOnyFLw9xOaRYVxa6MqXFGhLDUrBnNFhWBqVgwmtIzrpFDU\nzJ49m9dee41HHnmE1157jYiIiALdvmgeN0+KqtjYWLUy48aYjMwsTqWmczLlLCeSHf+eTD7LiRTH\njeHjSWfZ9ttptv1+GlXHfYAWNcJoU7siV9euwFVXhFPG38/bv0a+xcfHs337dm6++WYyMzNZv349\nrVq1ynN5EVmnqrH52ZdP3Mw2xpi09ExW7j7Owl8O8/2OI/x2Ki3PZcv4l6JCcAC1KpXl6Zvrc3Wt\nCjSrEebTieGcjIwMJk+ezMiRIwkLC2PPnj34+/u7TBJ/liUKY0yRdTL5LN9tP8KiXw6zLO4oKWcz\nKRvgx/X1K3FfbCgVygYQXjaACsEBhAX7O6aDAwgK8P2EkJs1a9bQv39/NmzYwO23386UKVPw9/d8\nM5klCmNMkZGVpew7kcK32w6z6JfDrN13kswspUq5QO5uEcktDavQtk5FAksXz0Tgytq1a2nTpg3V\nqlXjs88+o3v37gXSU8odliiMMYVKVTmSeIa9x5L59Vgye487/v31WAr7TiSTlp4FQHTVUAa2r8Mt\nDavQJLI8pQqxF1FRoars3LmTBg0a0LJlSyZPnsyDDz5IuXLlCjUOSxTGGJfSM7NISDl30/gsCbnc\nPE508TDaOarKoYQ0fj2eTMrZP2oP+fsJV1QIplZEWa6rF0HtSiFcWzeCKyoGe/LXKvLi4uIYNGgQ\nP/74I9u3b6dGjRoMGjTIK7FYojCmBEo+k0HckSR+S0jlZI4k8MfJP92ZEFwngeAAP8KDAxxPB7vR\nDFKlXCCta1WgVkTZ7Ff1sKBCfeagqDtz5gyvvfYar776KoGBgYwfP57q1at7NSZLFMYUY2cyMtlz\nNJmdhxPZ8Xui49/DiRw4cXHln3Mn/fCy/oQHB1CzQnD2zeEKZf0JCw7Inj63THHoRVSUnDp1itat\nW7Nz50569OjBP/7xD68nCbBEYUyxkXwmg19+O83m+FNsPniKLQdPsedYMpnO+hOlSwl1KoXQvEY4\nPWJrUK9KKFHhQdknfzvpe8+ZM2cIDAykfPnydOnShZtvvpmOHTt6O6xsliiMKQBHTqexfNexS1b1\nLGjpmVnsOJzI5vhT7D6alF2TqHJoIE0iy9OxUVXqVw2lQZVQakWUJaC0DWpZlGRlZfGvf/2Ll156\nie+++46GDRvy+uuvezusi1iiMCYfMrOUDQcS+H7HEZbsOMKWg6e9Fkul0ECaRpbn9ibVaBpVniaR\n5alcrozX4jHu2bRpEwMGDGDlypV06NChUJ6HyC9LFMa4IT0zi80HT7Fm7wl+2nuCn349wem0DPxK\nCS2vCGdopwZ0qF+ZyLCgQo1LSkE5H6tLVNKpKsOHD+eNN94gPDycDz/8kAcffLDQnonID0sUxlzg\ndFo6e44ms+tIEruPJrHxQAI/708g1TmcZO1KZbmjaTXa1omgfb1KlA+2E7Vxn4iQmJjII488wrhx\n46hQoYK3Q7okSxTGJySknGXJjiMs3naEDfsT8hwJ7M86m5F1Xsnp0qWE+lVC6dGqBlfXqkDslRWo\nFBrokX2b4mv//v089dRTDB06lDZt2jB58mRKlfKd+0WWKEyRo6r8fjqNHb8n8stvp/l+x1HWOUs5\nRIQE0rZORYL8PfNH5leqFDUrBlOnUgh1KpWlRoVg/P185w/aFC3p6elMmjSJv/3tb6gq3bp1o02b\nNj6VJMASRYmWlaWkpGdeNP/w6TTmbTjE6j3HPfbNPS9nM5U9R5POe8grumooT3Sow00xVWhaQks5\nGN+zatUq+vfvz6ZNm+jcuTNTpkyhZs2a3g4rXyxRFCN7jiax9dCle98kpJxl1Z4TrNxznBMXjOx1\njgg0iwojuJCrcAaWFro0q0501VDqV3G8wsv+uWEcjfGGL7/8kuPHjzNnzhzuuuuuIn2z+lJs4KIi\nat2+E8z+6QAZme79/8QnpLJm7wm3t1+tfBmuqRNB/SohlLrgAxwU4MfNMVWoWt66WBrjLlXl3//+\nN9WrV+fGG28kNTWVjIwMQkNDvR0aYAMX+bSzGVls++00O35P5MOVv3IowVFa4WRKOuXKlHa7R03Z\ngNL8tWMDboyujL+f628uQQGlqV6+jE9/wzGmKNmxYwcDBw5kyZIl9OzZkxtvvJGgoMLtKu1JligK\nWWJaOgu2Hmb/iRTS0jOZt+Egh0+fARxt8Z2bVkcEqpYvQ5+2V1I20P6LjCmq0tLSGDt2LOPGjSMo\nKIipU6fy+OOPezusAmdnoQJwIvmsy5u+O39PZOLiOFLSM4g7nMSZDEe9fRFoVyeCF+9oSFR4EM1r\nhNm3fGN8yIwZMxg1ahS9evViwoQJVK1a1dsheYQlCjeknM1g0S+Hs0/wOX2/4whfb/79ktuIDAui\nQdVQWl4RTtcWkbSwpGCMTzp8+DC7du2iXbt2PPbYYzRq1Ijrr7/e22F5VIlPFMvjjjFzzX4UF1cE\nh5PYdSQp1/dEoN/1takRnnd7ZGm/UnRqVNV67xjjw7Kysnj33XcZPnw4YWFh7Nq1C39//2KfJKAE\nJorfTqXy67EUAA4mpDJi7mbKlSlNeHDeJ/HA0qV4p/dVNI0qf9F7wQGlqWAJwJhibcOGDQwYMIDV\nq1dz00038c4771C6dMk5fZac3xQ4cCKF68YvOW9eSGBp5v/lOusKaozJ1erVq7nmmmuIiIjgk08+\noVevXiWu2bjYJ4q09EyOJZ1h3oZDvL5gBwC3Na7KQ22vBBwF3qpYSWZjTA6qyv79+6lZsyatWrVi\n7NixPP7444SHh3s7NK8o9onigfdXs3bfSQAiQgJ58uZ6PNjGNx+jN8Z43r59+xg8eDA//PAD27dv\np2rVqgwdOtTbYXlVsU4UCSlnWbvvJPWrhPBEh7p0bFSVoEIuSWGM8Q3p6em8+eabvPLKK4gIr7zy\nChEREd4Oq0go1oli+vK9ALx4R0Our1/Jy9EYY4qqEydO0L59e7Zs2ULXrl2ZNGkSV1xxhbfDKjKK\nZaI4mniGd5fuZtG2wzSsVs6ShDEmVxkZGZQuXZrw8HDatWvHmDFj6Nq1q7fDKnKKVaJIPZvJ/f9c\nxYYDCYCjR1PXdtW9HJUxpqhRVT766CNGjhzJkiVLqF27NtOmTfN2WEVWsUoUe48ls+FAAm1rV6RD\ng0r0b1/H2yEZY4qYbdu2MXDgQJYuXUrbtm1JT0/3dkhFnm8Ns+RCVpayeNthAJ69tb4lCWPMeVSV\nl156iWbNmrFx40beffddli9fToMGDbwdWpHn0UQhIp1EZIeI7BKRYbm8X15EvhKRjSKyVUT65ndf\nvd9fzT8W7QQgplq5PxG1MaY4EhH27dtHz5492bFjB/369fO5IUm9xWNHSUT8gLeB24CGwP0i0vCC\nxQYBv6hqM6ADMEFELrsexvK4Y6zcc5wmkeX5bEBbK81tjAHgt99+o1evXmzatAmA6dOn89FHH1G5\ncmUvR+ZbPJlOWwO7VHWPqp4FZgEXdidQIFQcz8OHACeADC7T6wu2A/BAmytodWWFPxW0Mcb3ZWZm\n8vbbbxMdHc2cOXPYsGEDQImqz1SQPJkoIoEDOabjnfNymgLEAIeAzcCTqnpRLW8R6Scia0Vk7dGj\nRy/a0cb4U3RrEUmPVtbv2ZiSbv369bRt25bBgwdz9dVXs2XLFh566CFvh+XTvN1A1xHYAFQHmgNT\nROSiGwyq+p6qxqpqbKVK5z8Tca78d6C/PXFtjIEPP/yQ/fv3M3PmTBYsWEDdunW9HZLP82SiOAjU\nyDEd5ZyXU19gjjrsAvYC0Zezk7eX7ALgiQ7Wy8mYkkhV+fzzz1mxYgUAY8aMYfv27fTs2bPEVXn1\nFE8mip+AeiJSy3mDuifw5QXL7AduAhCRKkADYI+7O1BV5v7syD01KgQXRMzGGB+yd+9eOnfuzD33\n3MOUKVMACA0NJSwszMuRFS8eSxSqmgEMBhYA24BPVXWriAwQkQHOxUYD14jIZuBb4HlVPebuPlLT\nMwGoUSHv0eWMMcXP2bNnGTt2LI0aNWLZsmW8+eabfPTRR94Oq9jyaBcAVf0a+PqCedNy/HwIuDW/\n2/9ywyEA+jjHljDGlAxTp05lxIgRdOvWjbfeeouoqChvh1Ss+XRfsXPjTNzT0j4kxhR3x44dY9++\nfbRs2ZL+/fsTHR1Nx44dvR1WieDtXk9/StyRJCLDgghzMd61Mca3qSozZswgOjqanj17kpmZSZky\nZSxJFCKfTRSZWUrc4URvh2GM8aCtW7fSvn17HnnkkeyH5/z8rCt8YfPZpqf/bfmdlLOZvHq3FfQy\npjhasWIF7du3p1y5crz//vv07dvXajN5ic8e9RW7j1GuTGm6NLPxJowpTg4fdlSBvvrqqxkxYgTb\nt2/n0UcftSThRT575NMzswgOKE2pUvZAjTHFwcGDB7n33ntp0qQJJ06cwM/Pj1deeYULqzGYwuez\niSIxLYOQMj7bcmaMccrMzGTSpEnExMQwf/58nnrqKUJCQrwdlsnBrTOt88nqK5xlNoqEo4lniAix\n3k7G+LJjx47RsWNH1q9fT8eOHXn77bepU8fK8RQ1l7yiEJE7cFR2XeScbi4icz0d2KVs/z2RWhH2\nrcMYX5SV5SgSXbFiRaKjo5k9ezbffPONJYkiyp2mp1HA1UACgKpuALxajvHw6TSSzmRQOTTQm2EY\nYy6TqjJ79mxiYmI4ePAgIsK///1v7rvvPivgV4S5kyjSVTXhgnnqiWDctWT7EQCqh5XxZhjGmMuw\ne/dubrvtNnr27EloaCiJifYclK9wJ1FsE5H7gFLOSrBvAqs8HJdL57JU+/o2nKExRV1WVhZjxoyh\ncePGrFixgkmTJrF69Wqioy9rRAHjRe4kisFASyALmAOcAZ70ZFDGmOKjVKlSrF+/njvvvJNt27bx\nl7/8xZ6u9jHuJIqOqvq8qrZwvoYBt3k6MGOM7zp69CiPPvooO3bsAGDmzJl8+umnREZeOBqy8QXu\nJIoXc5n3QkEHYozxfVlZWbz//vs0aNCAjz/+mJUrVwIQGGgdT3xZns9RiEhHoBMQKSL/yPFWORzN\nUF5zOjUdgOBAu3w1pqjYvHkzAwYMYMWKFVx33XVMmzaNhg0bejssUwBcPXB3BNgCpAFbc8xPBIZ5\nMqhLOZSQSrkypSlXxt+bYRhjcpg0aRI7duxgxowZ9OnTx7q7FiOi6rqnq4iUUdW0QornkmJjY7Vs\nj9c5lZrOhpH5HhzPGFMA/u///o/q1avTokULTpw4QVZWFhEREd4Oy+RCRNapamx+1nXnHkWkiMwS\nkU0isvPcKz87KygZmUqo1Xkyxmvi4+Pp3r07nTt35o033gCgQoUKliSKKXcSxQfADEBw9Hb6FJjt\nwZgu6UhiGrc3qebNEIwpkTIyMnjzzTeJiYnhm2++YezYscyYMcPbYRkPcydRBKvqAgBV3a2qL+LF\n7rHpmUp6phLg57OFb43xWW+99RbPPPMM119/PVu3bmXYsGEEBFhxzuLOnfabMyJSCtgtIgOAg0Co\nZ8PKW/KZDACuqhnurRCMKVESEhI4ePAgjRo1on///tStW5cuXbrYzeoSxJ2v5U8DZYEhQDvgceAR\nTwblijoLeNStZJVjjfEkVWXmzJlER0dz7733kpWVRUhICF27drUkUcJcMlGo6mpVTVTV/ar6oKp2\nAX71fGjGGG+Ji4ujY8eO9OrVixo1avDJJ5/YUKQlmMumJxFpBUQCy1X1mIg0Ap4HbgSiCiE+Y0wh\nW758OTfffDOBgYFMmTKFAQMGWG2mEi7PrwgiMhb4N9Ab+J+IvAwsATYC9QslOmNMoTl58iQArVu3\nZtCgQWzfvp1BgwZZkjAuryi6As1UNVVEKgAHgCaquqdwQjPGFIbDhw/z7LPP8v3337Nt2zZCQ0OZ\nMGGCt8MyRYirRsc0VU0FUNUTwE5LEsYUH1lZWbz77rtER0fz6aef8sgjj+Dvb2VxzMVcXVHUFpE5\nzp8FqJVjGlXt5tHIjDEec/ToUbp06cKqVavo0KEDU6dOtYGETJ5cJYruF0xP8WQg7srIVAQoG2gl\nPIy5XKqKiGSX2/joo4944IEHrLurcSnPs62qfluYgbgrLT2T6sH+VChrT4MacznmzZvHyy+/zMKF\nC6lUqRJfffWVt0MyPsLnOkafTsvgunqVvB2GMT5j//79dO3albvuuouMjAyOHj3q7ZCMj/FoohCR\nTiKyQ0R2iUiuY1iISAcR2SAiW0Vk6aW2qapUK1+m4IM1ppjJysrijTfeICYmhsWLFzN+/HjWr19v\ngwmZy+Z2Q7+IBKrqmctY3g94G7gFiAd+EpEvVfWXHMuEAe8AnVR1v4hUvtR2FazZyRg3iAgLFy7k\npptuYvLkydSsWdPbIRkfdckrChFpLSKbgTjndDMRmezGtlsDu1R1j6qeBWbheDYjp17AHFXdD6Cq\nR9wJuqpdURiTq5MnTzJ48GD27duHiPDFF1/w5ZdfWpIwf4o7TU+TgM7AcQBV3Qjc4MZ6kTge0jsn\n3jkvp/pAuIh8LyLrROQhN7ZrPTSMuYCq8sknn9CgQQOmTZvGkiVLAAgODvZyZKY4cKfpqZSq7rvg\n5JxZgPtvCdwEBAErRWSVqp43gp6I9AP6AQRUrUvpUpYojDlnx44dPPHEE3z33XdcffXVLFy4kObN\nm3s7LFOMuHNFcUBEWgMqIn4i8hTgzlCoB4EaOaajnPNyigcWqGqyqh4DlgHNLtyQqr6nqrHnxnut\naPcojMk2evRo1q1bx9SpU/nxxx8tSZgCJ6rqegHHDeZJwM3OWYuBwc4Tu6v1SuNIKDfhSBA/Ab1U\ndWuOZWJwPMjXEQgA1gA9VXVLXtsNrFZPt25cT93KXhs7yRivW7RoEVFRUcTExHDkyBGysrKoWrWq\nt8MyRZiIrDv3ZftyuXNFkaGqPVU1wvnqeakkAaCqGcBgYAGwDfhUVbeKyADnSHmo6jbgf8AmHEni\nfVdJ4hw/q4tvSqjff/+dXr16ceuttzJ27FgAKleubEnCeJQ7VxS7gR3AbBw9lBILI7C8BFarpzu3\nbKBmxbLeDMOYQpWZmcm7777LiBEjSE1NZfjw4QwbNowyZawHoHGPR68oVLUOMAbHTefNIvKFiPTM\nz84KggBR4daTw5Qsr7/+OoMGDSI2NpbNmzfz8ssvW5IwheaSVxTnLewYl2Ii0FtVvTKaSVD1+pp6\nyJ176cb4tsTERA4fPkzdunVJSEjgm2++oWfPntY93OSLR68oRCRERHqLyFc47iMcBa7Jz86MMZem\nqsyZM4eYmBjuu+8+VJWwsDDuv/9+SxLGK9y5K7wFaAOMV9W6qvqsqq72cFzGlEi//vorXbp0oXv3\n7kRERPDOO+9YcjBe584Dd7VVNcvjkRhTwi1btoxOnTpRqlQpJkyYwJAhQyhd2sZdMd6X56dQRCao\n6rPA5yJy0Y0MG+HOmIKRlJRESEgIsbGx9OnThxEjRlCjRo1Lr2hMIcnzZraItFbVNSJyU27ve2tg\nI7uZbYqL48ePM2zYMJYsWcLmzZsJCgrydkimGPPIzWxVXeP8MUZVv835AmLyszNjjONm9Ycffkh0\ndDQzZsygW7duXE7vQ2MKmzs3sx/JZd6jBR2IMSXBkSNHuOGGG3j44YepV68e69evZ/z48Vbl1RRp\nru5R9AB6ArVEZE6Ot0KBBE8HZkxxFB4eDsB7773Ho48+SikrR2N8gKsuFWtwjEERhWOkunMSgZ89\nGZQxxcmCBQsYPXo08+fPJywsjCVLlliXV+NT8kwUqroX2IujWqwx5jIdOnSIp59+mk8//ZT69etz\n8OBBwsLCLEkYn5Pnda+ILHX+e1JETuR4nRSRE4UXojG+JSsriylTphATE8O8efMYNWoUmzZtolGj\nRt4OzZh8cdU9tpSqZolIrjWdVLWgRrm7LNY91hR1WVlZtG/fnqCgIN555x3q1q3r7ZCM8Vj32HNP\nY9cA/JyJoS3QH7Aa38bkcPr0af7617/y22+/UapUKebPn8+CBQssSZhiwZ0uF1/gGAa1DjADqAf8\nx6NRGeMjVJXPPvuM6OhoJkyYwMKFCwEoX7683YswxYY7iSJLVdOBbsBkVX0aiPRsWMYUfXv27OGO\nO+7gvvvuo2rVqqxevZo+ffp4OyxjCpxbQ6GKyL3Ag8B85zx/z4VkjG8YOnQoP/zwAxMnTmTNmjW0\natXK2yEZ4xHulKZ8BHgCR5nxPSJSC5jp2bCMKZqWLVtGVFQUtWvXZuLEiQBERUV5OSpjPMudoVC3\nAEOAtSISDRxQ1b97PDJjipBjx47xyCOP0L59e8aMGQM4EoQlCVMSuDPC3XXALuBfwHRgp4i083Rg\nxhQFWVlZTJ8+nQYNGvDxxx8zbNgwpkyZ4u2wjClU7jQ9vQncrqq/AIhIDPAxkK/+uMb4krFjx/Li\niy/Srl07pk2bRuPGjb0dkjGFzp1EEXAuSQCo6jYRCfBgTMZ4VUpKCseOHeOKK67g8ccfp3r16vTp\n08cK+JkSy51P/noRmSYi1zpfU7GigKaY+vrrr2nUqBE9evRAValcuTJ9+/a1JGFKNHc+/QOAPcBQ\n52sPjqezjSk2Dh48yL333ssdd9xBUFAQ48aNswfmjHFy2fQkIk2AOsBcVR1fOCEZU7iWLl1K586d\nycjIYMx5x1i+AAAd1UlEQVSYMfz1r38lIMBaV405x1X12BE4ynf0BhaJSG4j3Rnjs9LS0gBo0aIF\nd999N1u2bOGFF16wJGHMBVxVj90KtFbVZBGpBHytql5/9NSqx5o/69SpU7zwwgssXbqUdevWWWIw\nJYJHqscCZ1Q1GUBVj15iWWOKPFVl9uzZREdHM3XqVG644QbS09O9HZYxRZ6rexS1c4yVLUCdnGNn\nq2o3j0ZmTAE6cuQIDz30EAsWLKBly5Z89dVXxMbao0DGuMNVouh+wbQ9jmp8Vrly5Th+/DiTJ09m\n4MCB+PnlOh6XMSYXrsbM/rYwAzGmoH3//feMGzeOzz//nLJly7J69Wp7HsKYfLC/GlPsHD16lD59\n+nDDDTcQFxfHvn37ACxJGJNPHv3LEZFOIrJDRHaJyDAXy7USkQwRuceT8ZjiLSsri/fff58GDRow\nc+ZMXnjhBbZs2ULDhg29HZoxPs2dWk8AiEigqp65jOX9gLeBW4B44CcR+TJn3agcy70GLHR328bk\nRlWZOnUqTZs2ZerUqcTExHg7JGOKBXfKjLcWkc1AnHO6mYhMdmPbrYFdqrpHVc8Cs4CuuSz3F+Bz\n4Ij7YRvjkJyczMiRIzl+/Dh+fn4sWLCAJUuWWJIwpgC50/Q0CegMHAdQ1Y3ADW6sFwkcyDEdzwVj\nbYtIJHA3MNXVhkSkn4isFZG1eT0gaEqer776ioYNGzJ69Gjmz3eM0hsREWE1mowpYO4kilKquu+C\neZkFtP+JwPOqmuVqIVV9T1VjVTXWTgLmwIEDdOvWjS5duhASEsKyZcvo06ePt8Mypthy5x7FARFp\nDajzfsJfAHdqaBwEauSYjnLOyykWmOU8+UcAt4tIhqp+4cb2TQn1xBNP8O233zJ27FieeeYZK8Fh\njIflWespewGRyjian252zloMDFbVY5dYrzSOhHITjgTxE9BLVbfmsfwHwHxV/a+r7Vqtp5Jp9erV\nREZGEhUVxZ49exARatWq5e2wjPEZnqr1BICqHlHVnqoa4Xz1vFSScK6XAQwGFgDbgE9VdauIDBCR\nAfkJ1pQ8CQkJDBw4kLZt2zJq1CgAateubUnCmELkzhXFP4GLFlLVfp4KyhW7oigZVJVZs2bx9NNP\nc/ToUYYMGcKoUaMIDQ31dmjG+KQ/c0Xhzj2KxTl+LoOjl9KBPJY1pkD8/e9/56WXXqJVq1Z88803\ntGjRwtshGVNiXTJRqOrsnNMi8jGw3GMRmRLrzJkznDx5kqpVq/Lwww9ToUIF+vfvbwX8jPGy/JTw\nqAVUKehATMn27bff0rRpU3r37o2qEhUVxRNPPGFJwpgiwJ0ns0+KyAnnKwFYBAz3fGimJDh8+DAP\nPPAAN998M5mZmTz//PP2wJwxRYzLpidx/MU244/nH7LUHo02BWTp0qXcddddJCcn89JLLzF8+HCC\ngoK8HZYx5gIuE4Wqqoh8raqNCysgU/xlZGRQunRpGjduzE033cSYMWOIjo72dljGmDy4c49ig4hY\nlxPzpyUlJfHcc89x7bXXkpmZScWKFfnvf/9rScKYIi7PROF8shqgBY4S4TtEZL2I/Cwi6wsnPFNc\nfPHFF8TExDBhwgSaNm1KWlqat0MyxrjJVdPTGuAqoEshxWKKoSNHjvD444/z5Zdf0qRJE2bPns01\n11zj7bCMMZfBVaIQAFXdXUixmGKobNmyxMXFMX78eJ566in8/f29HZIx5jK5ShSVROSZvN5U1X94\nIB5TDKxYsYI33niD//znP5QtW5ZNmzZRurTbgykaY4oYVzez/YAQIDSPlzHnOXHiBP3796ddu3b8\n9NNP7NmzB8CShDE+ztVf8G+qOqrQIjE+S1X55JNPePbZZzlx4gTPPvssL7/8MiEhId4OzRhTAC55\nj8KYS0lPT2fs2LHUqVOHRYsW0axZM2+HZIwpQK6anm4qtCiMz0lLS2PcuHGcPn2agIAAFi1axI8/\n/mhJwphiKM9EoaonCjMQ4zsWLVpEkyZNGD58OPPmzQMgMjKSUqXyU2PSGFPU2V+2cdvvv/9Or169\nuPXWWxERFi9ezIMPPujtsIwxHmbdUYzbHn74YZYsWcLLL7/M888/T5kyZbwdkjGmEFiiMC79/PPP\nREVFUalSJSZOnEipUqWoX7++t8MyxhQia3oyuUpMTOTpp58mNjaWV155BYDo6GhLEsaUQHZFYc6j\nqsydO5chQ4Zw6NAh+vfvz+jRo70dljHGi+yKwpxn9OjRdO/enYiICFasWMHUqVMJDw/3dljGGC+y\nKwpDeno6p0+fpmLFivTu3ZuQkBCGDBlipTeMMYBdUZR4y5cvp0WLFjz88MMA1KlTh2eeecaShDEm\nmyWKEur48eM89thjXHfddSQmJtKvXz9vh2SMKaLsa2MJtHTpUu655x5OnjzJ0KFDGTlyJGXLlvV2\nWMaYIsoSRQmSlZWV/RxEbGws48ePp0mTJt4OyxhTxImqejuGyxJUvb6mHtrp7TB8SmpqKn//+99Z\ntWoVCxcutJpMxpRAIrJOVWPzs66dMYq5//3vfzRu3Ji///3vREZGkpqa6u2QjDE+xhJFMXX06FF6\n9OjBbbfdhr+/P9999x0ffvih3Yswxlw2SxTFlL+/P2vWrGH06NFs3LiRG264wdshGWN8lN3MLkbW\nrVvHW2+9xb/+9S/CwsLYsWMHAQEB3g7LGOPjPHpFISKdRGSHiOwSkWG5vN9bRDaJyGYRWSEiNjxa\nPpw+fZonn3yS1q1bs2jRInbt2gVgScIYUyA8lihExA94G7gNaAjcLyINL1hsL9BeVZsAo4H3PBVP\ncaSqfPbZZ0RHRzN58mQGDhzItm3biImJ8XZoxphixJNNT62BXaq6B0BEZgFdgV/OLaCqK3IsvwqI\n8mA8xc7Zs2cZNmwYVapU4YsvvqB169beDskYUwx5sukpEjiQYzreOS8vjwLf5PaGiPQTkbUistbX\nnvsoaGfPnuWtt94iJSWFwMBAFi9ezE8//WRJwhjjMUWi15OI3IAjUTyf2/uq+p6qxqpqrIgUbnBF\nyLJly2jevDlPPfUUc+fOBaBWrVpWwM8Y41GeTBQHgRo5pqOc884jIk2B94Guqnrcg/H4rGPHjtG3\nb1/at29Pamoq8+fPp3fv3t4OyxhTQnjyq+hPQD0RqYUjQfQEeuVcQESuAOYAD6qq1eXIQ48ePVi2\nbBnDhw/nxRdfJDg42NshGWNKEI8lClXNEJHBwALAD5iuqltFZIDz/WnASKAi8I6zSSkjv7VIiput\nW7cSGRlJWFgYEyZMwN/fn0aNGnk7LGNMCWRFAYuYlJQURo8ezRtvvMGQIUOYMGGCt0MyxhQDf6Yo\noN0FLUK+/vprBg0axK+//krfvn0ZPny4t0Myxpii0evJwKhRo7jjjjsICgpi6dKlTJ8+nYiICG+H\nZYwxdkXhTRkZGaSkpFCuXDm6d+9O6dKlee6556z0hjGmSLFE4SVr1qxhwIABNGjQgJkzZ9KoUSO7\nWW2MKZKs6amQnTp1isGDB9OmTRt+//137r77bm+HZIwxLtkVRSFaunQpPXv25MiRIwwePJgxY8ZQ\nrlw5b4dljDEuWaIoBKqKiFCzZk3q16/PV199RWysPS5ijPEN9hyFB505c4bx48ezbt065s6dS0mu\nU2WM8a4/8xyF3aPwkCVLltCsWTNGjhxJYGAgaWlp3g7JGGPyxRJFATt+/Dh9+vThxhtvJD09nW++\n+YbZs2cTFBTk7dCMMSZfLFEUsMzMTBYsWMALL7zAli1b6NSpk7dDMsaYP8VuZheAzZs3M23aNCZP\nnkzlypXZvXs3ZcuW9XZYxhhTIOyK4k9ITk5m6NChtGjRgk8//ZS4uDgASxLGmGLFrijy6auvvmLw\n4MHs37+fxx57jHHjxlGxYkVvh2W8ID09nfj4eOuwYIqEMmXKEBUVhb+/f4Ft0xJFPqSmpjJw4EDC\nwsL44YcfuPbaa70dkvGi+Ph4QkNDufLKK60LtPEqVeX48ePEx8dTq1atAtuuNT25KSMjg/fff5+z\nZ88SFBTE4sWL+fnnny1JGNLS0qhYsaIlCeN1IkLFihUL/OrWEoUbVq1aRWxsLI8//jhz5swBIDo6\nukAv7YxvsyRhigpPfBYtUbhw8uRJBgwYwDXXXMOxY8f4/PPP6dGjh7fDMsaYQmWJwoWuXbvyz3/+\nk6eeeopt27bRrVs3++ZoiiQ/Pz+aN29O48aNufPOO0lISMh+b+vWrdx44400aNCAevXqMXr0aHKW\n7vnmm2+IjY2lYcOGtGjRgmeffdYbv4JLP//8M48++qi3w3Bp7Nix1K1blwYNGrBgwYJcl9m4cSNt\n27alSZMm3HnnnZw+fRqARYsW0bJlS5o0aULLli357rvvstd54YUXqFGjBiEhIedta8qUKUyfPt1z\nv1BOqupTrzLV6qkn7dy5UxMTE1VVdeXKlbp+/XqP7s/4vl9++cXbIWjZsmWzf37ooYd0zJgxqqqa\nkpKitWvX1gULFqiqanJysnbq1EmnTJmiqqqbN2/W2rVr67Zt21RVNSMjQ995550CjS09Pf1Pb+Oe\ne+7RDRs2FOo+L8fWrVu1adOmmpaWpnv27NHatWtrRkbGRcvFxsbq999/r6qq//rXv/TFF19UVdX1\n69frwYMHVdXxf1K9evXsdVauXKmHDh067/9Y1fF/2bx581zjye0zCazVfJ53rdeTU1paGq+99hqv\nvvoqzz77LK+++ipt2rTxdljGx7zy1VZ+OXS6QLfZsHo5/nan+4NatW3blk2bNgHwn//8h3bt2nHr\nrbcCEBwczJQpU+jQoQODBg1i/PjxvPDCC0RHRwOOK5OBAwdetM2kpCT+8pe/sHbtWkSEv/3tb3Tv\n3p2QkBCSkpIA+O9//8v8+fP54IMPePjhhylTpgw///wz7dq1Y86cOWzYsIGwsDAA6tWrx/LlyylV\nqhQDBgxg//79AEycOJF27dqdt+/ExEQ2bdpEs2bNAMegX08++SRpaWkEBQUxY8YMGjRowAcffMCc\nOXNISkoiMzOTpUuX8vrrr/Ppp59y5swZ7r77bl555RUA7rrrLg4cOEBaWhpPPvkk/fr1c/v45mbe\nvHn07NmTwMBAatWqRd26dVmzZg1t27Y9b7mdO3dy/fXXA3DLLbfQsWNHRo8eTYsWLbKXadSoEamp\nqZw5c4bAwMA8z0PBwcFceeWVrFmzhtatW/+p+C/FEgWwePFinnjiCeLi4rj//vsZMmSIt0MyJl8y\nMzP59ttvs5tptm7dSsuWLc9bpk6dOiQlJXH69Gm2bNniVlPT6NGjKV++PJs3bwYc9+8uJT4+nhUr\nVuDn50dmZiZz586lb9++rF69mpo1a1KlShV69erF008/zbXXXsv+/fvp2LEj27ZtO287a9eupXHj\nxtnT0dHR/PDDD5QuXZrFixczYsQIPv/8cwDWr1/Ppk2bqFChAgsXLiQuLo41a9agqnTp0oVly5Zx\n/fXXM336dCpUqEBqaiqtWrWie/fuFz0H9fTTT7NkyZKLfq+ePXsybNiw8+YdPHjwvBN6VFQUBw8e\nvGjdRo0aMW/ePO666y4+++wzDhw4cNEyn3/+OVdddRWBgYGXPMaxsbH88MMPlig8bfTo0YwcOZK6\ndeuycOFCbrnlFm+HZHzY5XzzL0ipqak0b96cgwcPEhMTU+Cf48WLFzNr1qzs6fDw8Euuc++99+Ln\n5wdAjx49GDVqFH379mXWrFnZnUIWL17ML7/8kr3O6dOnSUpKOq89/rfffqNSpUrZ06dOnaJPnz7E\nxcUhIqSnp2e/d8stt1ChQgUAFi5cyMKFC7O/rSclJREXF8f111/PpEmTmDt3LgAHDhwgLi7uokTx\n5ptvundwLsP06dMZMmQIo0ePpkuXLgQEBJz3/tatW3n++edZuHChW9urXLky27dvL/A4L1QiE0VW\nVhZpaWkEBwdz2223kZ6ezogRIyhTpoy3QzMmX4KCgtiwYQMpKSl07NiRt99+myFDhtCwYUOWLVt2\n3rJ79uwhJCSEcuXK0ahRI9atW5fdrHO5cnbuuLDvfs5SNm3btmXXrl0cPXqUL774ghdffBFw/C2u\nWrXK5d9eUFDQedt+6aWXuOGGG5g7dy6//vorHTp0yHWfqsrw4cPp37//edv7/vvvWbx4MStXriQ4\nOJgOHTrk+tzB5VxRREZGnnd1EB8fT2Rk5EXrRkdHZyeBnTt38n//93/nrXP33Xfz0UcfUadOnbwO\nx3nONb95Wonr9bRx40batWuX3bwUGxvLqFGjLEmYYiE4OJhJkyYxYcIEMjIy6N27N8uXL2fx4sWA\n48pjyJAhDB06FIC//vWvvPrqq+zc6RgMLCsri2nTpl203VtuuYW33347e/pc01OVKlXYtm0bWVlZ\n2d/QcyMi3H333TzzzDPExMRkf3u/9dZbmTx5cvZyGzZsuGjdmJgYdu3alT196tSp7JPwBx98kOc+\nO3bsyPTp07PvoRw8eJAjR45w6tQpwsPDCQ4OZvv27axatSrX9d988002bNhw0evCJAHQpUsXZs2a\nxZkzZ9i7dy9xcXG5NgcdOXIEcBznMWPGMGDAAAASEhK44447GDdu3EX3aFzZuXPnec1ynlJiEkVS\nUhLPPfccLVu2ZPfu3ed9CzGmOGnRogVNmzZl5syZBAUFMW/ePMaMGUODBg1o0qQJrVq1YvDgwQA0\nbdqUiRMncv/99xMTE0Pjxo3Zs2fPRdt88cUXOXnyJI0bN6ZZs2bZ37THjRtH586dueaaa6hWrZrL\nuHr06MEnn3xy3rNIkyZNYu3atTRt2pSGDRvmmqSio6M5deoUiYmJAAwdOpThw4fTokULMjIy8tzf\nrbfeSq9evbK7o95zzz0kJibSqVMnMjIyiImJYdiwYQXSaaVRo0bcd999NGzYkE6dOvH2229nN7s9\n9thjrF27FoCZM2dSv359oqOjqV69On379gUcXV137drFqFGjaN68Oc2bN89OKkOHDiUqKoqUlBSi\noqJ4+eWXs/f7448/FkpzeYkYCvWHH36gV69exMfH079/f8aOHetWG6sx7ti2bRsxMTHeDqNYe/PN\nNwkNDeWxxx7zdihFxs8//8w//vEPPv7444vey+0zaUOhXkLlypWpVq0aK1asYNq0aZYkjPExAwcO\ndKsXUEly7NgxRo8eXSj7KpZXFOnp6UycOJEtW7bw4YcfAo4bW/ZUtfEEu6IwRY1dUVzCihUraNmy\nJUOHDuXkyZPZvRksSRhP8rUvXKb48sRnsdgkipMnT9KvXz/atWtHQkICX3zxBV9++aX1ZjIeV6ZM\nGY4fP27JwnidOsejKOjzXrF5jiI5OZnPPvuMZ599lpdffvmiAlrGeEpUVBTx8fEcPXrU26EYkz3C\nXUHy6USxfft2ZsyYwbhx44iKimLv3r3ZtWSMKSz+/v4FOpqYMUWNR5ueRKSTiOwQkV0ictFTKuIw\nyfn+JhG5yp3tpqamMnLkSJo2bcp7773H7t27ASxJGGOMB3jsikJE/IC3gVuAeOAnEflSVX/Jsdht\nQD3n62pgqvPfPGWdSaZJkybs3r2bBx54gDfeeIMqVap45pcwxhjj0aan1sAuVd0DICKzgK5AzkTR\nFfjIWSt9lYiEiUg1Vf0tr42mJxymVMXaLF68mJtuusmD4RtjjAHPJopIIGcN3XguvlrIbZlI4LxE\nISL9gHMF48/ExcVtufnmmws2Wt8UARzzdhBFhB2LP9ix+IMdiz80yO+KPnEzW1XfA94DEJG1+X1o\npLixY/EHOxZ/sGPxBzsWfxCRtfld15M3sw8CNXJMRznnXe4yxhhjvMiTieInoJ6I1BKRAKAn8OUF\ny3wJPOTs/dQGOOXq/oQxxpjC57GmJ1XNEJHBwALAD5iuqltFZIDz/WnA18DtwC4gBejrxqbf81DI\nvsiOxR/sWPzBjsUf7Fj8Id/HwueKAhpjjClcxabWkzHGGM+wRGGMMcalIpsoPFX+wxe5cSx6O4/B\nZhFZISLNvBFnYbjUscixXCsRyRCRewozvsLkzrEQkQ4iskFEtorI0sKOsbC48TdSXkS+EpGNzmPh\nzv1QnyMi00XkiIhsyeP9/J03VbXIvXDc/N4N1AYCgI1AwwuWuR34BhCgDbDa23F78VhcA4Q7f76t\nJB+LHMt9h6OzxD3ejtuLn4swHJUQrnBOV/Z23F48FiOA15w/VwJOAAHejt0Dx+J64CpgSx7v5+u8\nWVSvKLLLf6jqWeBc+Y+csst/qOoqIExEXI/u7psueSxUdYWqnnROrsLxPEpx5M7nAuAvwOfAkcIM\nrpC5cyx6AXNUdT+AqhbX4+HOsVAgVBwjmIXgSBQZhRum56nqMhy/W17ydd4sqokir9Iel7tMcXC5\nv+ejOL4xFEeXPBYiEgncjaPAZHHmzueiPhAuIt+LyDoReajQoitc7hyLKUAMcAjYDDypqlmFE16R\nkq/zpk+U8DDuEZEbcCSKa70dixdNBJ5X1Swb/pbSQEvgJiAIWCkiq1TV9aDzxVNHYANwI1AHWCQi\nP6jqae+G5RuKaqKw8h9/cOv3FJGmwPvAbap6vJBiK2zuHItYYJYzSUQAt4tIhqp+UTghFhp3jkU8\ncFxVk4FkEVkGNAOKW6Jw51j0Bcapo6F+l4jsBaKBNYUTYpGRr/NmUW16svIff7jksRCRK4A5wIPF\n/NviJY+FqtZS1StV9Urgv8ATxTBJgHt/I/OAa0WktIgE46jevK2Q4ywM7hyL/TiurBCRKjgqqe4p\n1CiLhnydN4vkFYV6rvyHz3HzWIwEKgLvOL9JZ2gxrJjp5rEoEdw5Fqq6TUT+B2wCsoD3VTXXbpO+\nzM3PxWjgAxHZjKPHz/OqWuzKj4vITKADECEi8cDfAH/4c+dNK+FhjDHGpaLa9GSMMaaIsERhjDHG\nJUsUxhhjXLJEYYwxxiVLFMYYY1yyRGGKHBHJdFY8Pfe60sWyV+ZVKfMy9/m9s/roRhH5UUQa5GMb\nA86VyRCRh0Wkeo733heRhgUc508i0tyNdZ5yPkdhTL5YojBFUaqqNs/x+rWQ9ttbVZsBHwKvX+7K\nzmcXPnJOPgxUz/HeY6r6S4FE+Uec7+BenE8BlihMvlmiMD7BeeXwg4isd76uyWWZRiKyxnkVsklE\n6jnnP5Bj/rsi4neJ3S0D6jrXvUlEfhbHWB/TRSTQOX+ciPzi3M8bznkvi8hz4hgDIxb4t3OfQc4r\ngVjnVUf2yd155TEln3GuJEdBNxGZKiJrxTHewivOeUNwJKwlIrLEOe9WEVnpPI6fiUjIJfZjSjhL\nFKYoCsrR7DTXOe8IcIuqXgX0ACblst4A4C1VbY7jRB0vIjHO5ds552cCvS+x/zuBzSJSBvgA6KGq\nTXBUMhgoIhVxVKhtpKpNgTE5V1bV/wJrcXzzb66qqTne/ty57jk9cNSmyk+cnYCc5UlecD6R3xRo\nLyJNVXUSjoqpN6jqDSISAbwI3Ow8lmuBZy6xH1PCFckSHqbES3WeLHPyB6Y42+QzcZTQvtBK4AUR\nicIxDkOciNyEo4LqT87yJkHkPU7Fv0UkFfgVx5gWDYC9OepnfQgMwlGyOg34l4jMB+a7+4up6lER\n2eOssxOHozDdj87tXk6cATjGVch5nO4TkX44/q6rAQ1xlO/IqY1z/o/O/QTgOG7G5MkShfEVTwOH\ncVQ/LYXjRH0eVf2PiKwG7gC+FpH+OOr6fKiqw93YR29VXXtuQkQq5LaQs7ZQaxxF5u4BBuMoX+2u\nWcB9wHZgrqqqOM7abscJrMNxf2Iy0E1EagHPAa1U9aSIfACUyWVdARap6v2XEa8p4azpyfiK8sBv\nzsFmHsRR/O08IlIb2ONsbpmHownmW+AeEansXKaCiNR0c587gCtFpK5z+kFgqbNNv7yqfo0jgeU2\nRnkiEJrHdufiGGnsfhxJg8uN01ku+yWgjYhEA+WAZOCUOKqj3pZHLKuAdud+JxEpKyK5XZ0Zk80S\nhfEV7wB9RGQjjuaa5FyWuQ/YIiIbgMY4hnz8BUeb/EIR2QQswtEsc0mqmoajuuZnzqqjWcA0HCfd\n+c7tLSf3Nv4PgGnnbmZfsN2TOMp911TVNc55lx2n897HBOCvqroR+BnHVcp/cDRnnfMe8D8RWaKq\nR3H0yJrp3M9KHMfTmDxZ9VhjjDEu2RWFMcYYlyxRGGOMcckShTHGGJcsURhjjHHJEoUxxhiXLFEY\nY4xxyRKFMcYYl/4fJKhdZPmlUccAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114f16630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Determine the false positive and true positive rates\n",
    "fpr, tpr, _ = roc_curve(target_test, rf.predict_proba(features_test)[:,1]) \n",
    "    \n",
    "# Calculate the AUC\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('ROC AUC: %0.3f' % roc_auc)\n",
    " \n",
    "# Plot of a ROC curve for a specific class\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Random Forest does the best, but I still am not getting the accurancy on my target class of interest. I have a few tricks I can do to work on this, but that is for another day/class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Gradient Boost Classification\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "clf_GBC = GradientBoostingClassifier(n_estimators=100, learning_rate=1.7, max_depth=1, random_state=0)\n",
    "clf_GBC.fit(features_train, target_train)\n",
    "predicted_GBC=clf_GBC.predict(features_test)\n",
    "expected = target_test\n",
    "print(classification_report(expected, predicted_GBC,target_names=['No', 'Yes']))\n",
    "print(confusion_matrix(expected, predicted_GBC))\n",
    "print(accuracy_score(expected,predicted_GBC))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#AdaBoost of a Decision Tree\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=3),\n",
    "                         algorithm=\"SAMME\",\n",
    "                         n_estimators=200)\n",
    "bdt.fit(features_train, target_train)\n",
    "predicted_bdt=bdt.predict(features_test)\n",
    "expected = target_test\n",
    "print(classification_report(expected, predicted_bdt,target_names=['No', 'Yes']))\n",
    "print(confusion_matrix(expected, predicted_bdt))\n",
    "print(accuracy_score(expected,predicted_bdt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sknn.mlp import Classifier, Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nn = Classifier(\n",
    "    layers=[\n",
    "        Layer(\"Sigmoid\", units=100),\n",
    "        Layer(\"Softmax\")],\n",
    "    learning_rate=0.02,\n",
    "    n_iter=10)\n",
    "nn.fit(features_train, target_train)\n",
    "\n",
    "y_valid = nn.predict(features_test)\n",
    "\n",
    "score = nn.score(features_test, target_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}