Week 4/code/.ipynb_checkpoints/Churn Ensembles Example in Python-PM class-new-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensembles, Tuning and CV\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Like R, Python uses packages in data mining/machine learning. The 3 mose common ones are Pandas (manipulation), Scikit Learn (machine learning) and Matplotlit (graphics)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/mylesgartland/OneDrive - Rockhurst University/Courses/Predictive Models/Pred_Models_git/Week 4/code'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, KFold, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import roc_curve, auc, confusion_matrix, classification_report\n",
    "import time\n",
    "from operator import itemgetter\n",
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/mylesgartland/OneDrive - Rockhurst University/Courses/Predictive Models/Pred_Models_git/Week 4\n"
     ]
    }
   ],
   "source": [
    "cd '/Users/mylesgartland/OneDrive - Rockhurst University/Courses/Predictive Models/Pred_Models_git/Week 4/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in Data\n",
    "# Churn Calls Data\n",
    "This is a Pandas operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state</th>\n",
       "      <th>account_length</th>\n",
       "      <th>area_code</th>\n",
       "      <th>international_plan</th>\n",
       "      <th>voice_mail_plan</th>\n",
       "      <th>number_vmail_messages</th>\n",
       "      <th>total_day_minutes</th>\n",
       "      <th>total_day_calls</th>\n",
       "      <th>total_day_charge</th>\n",
       "      <th>total_eve_minutes</th>\n",
       "      <th>total_eve_calls</th>\n",
       "      <th>total_eve_charge</th>\n",
       "      <th>total_night_minutes</th>\n",
       "      <th>total_night_calls</th>\n",
       "      <th>total_night_charge</th>\n",
       "      <th>total_intl_minutes</th>\n",
       "      <th>total_intl_calls</th>\n",
       "      <th>total_intl_charge</th>\n",
       "      <th>number_customer_service_calls</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AK</td>\n",
       "      <td>1</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>175.2</td>\n",
       "      <td>74</td>\n",
       "      <td>29.78</td>\n",
       "      <td>151.7</td>\n",
       "      <td>79</td>\n",
       "      <td>12.89</td>\n",
       "      <td>230.5</td>\n",
       "      <td>109</td>\n",
       "      <td>10.37</td>\n",
       "      <td>5.3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.43</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>30</td>\n",
       "      <td>146.3</td>\n",
       "      <td>128</td>\n",
       "      <td>24.87</td>\n",
       "      <td>162.5</td>\n",
       "      <td>80</td>\n",
       "      <td>13.81</td>\n",
       "      <td>129.3</td>\n",
       "      <td>109</td>\n",
       "      <td>5.82</td>\n",
       "      <td>14.5</td>\n",
       "      <td>6</td>\n",
       "      <td>3.92</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>19</td>\n",
       "      <td>171.9</td>\n",
       "      <td>96</td>\n",
       "      <td>29.22</td>\n",
       "      <td>198.4</td>\n",
       "      <td>111</td>\n",
       "      <td>16.86</td>\n",
       "      <td>321.7</td>\n",
       "      <td>76</td>\n",
       "      <td>14.48</td>\n",
       "      <td>10.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.84</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AK</td>\n",
       "      <td>41</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>159.3</td>\n",
       "      <td>66</td>\n",
       "      <td>27.08</td>\n",
       "      <td>125.9</td>\n",
       "      <td>75</td>\n",
       "      <td>10.70</td>\n",
       "      <td>261.9</td>\n",
       "      <td>76</td>\n",
       "      <td>11.79</td>\n",
       "      <td>11.1</td>\n",
       "      <td>5</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AK</td>\n",
       "      <td>42</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>129</td>\n",
       "      <td>29.07</td>\n",
       "      <td>183.9</td>\n",
       "      <td>96</td>\n",
       "      <td>15.63</td>\n",
       "      <td>130.2</td>\n",
       "      <td>90</td>\n",
       "      <td>5.86</td>\n",
       "      <td>4.6</td>\n",
       "      <td>6</td>\n",
       "      <td>1.24</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AK</td>\n",
       "      <td>48</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>37</td>\n",
       "      <td>211.7</td>\n",
       "      <td>115</td>\n",
       "      <td>35.99</td>\n",
       "      <td>159.9</td>\n",
       "      <td>84</td>\n",
       "      <td>13.59</td>\n",
       "      <td>144.1</td>\n",
       "      <td>80</td>\n",
       "      <td>6.48</td>\n",
       "      <td>12.2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.29</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AK</td>\n",
       "      <td>50</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>183.6</td>\n",
       "      <td>107</td>\n",
       "      <td>31.21</td>\n",
       "      <td>58.6</td>\n",
       "      <td>118</td>\n",
       "      <td>4.98</td>\n",
       "      <td>202.6</td>\n",
       "      <td>99</td>\n",
       "      <td>9.12</td>\n",
       "      <td>8.7</td>\n",
       "      <td>3</td>\n",
       "      <td>2.35</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AK</td>\n",
       "      <td>51</td>\n",
       "      <td>area_code_510</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>12</td>\n",
       "      <td>135.8</td>\n",
       "      <td>60</td>\n",
       "      <td>23.09</td>\n",
       "      <td>200.6</td>\n",
       "      <td>134</td>\n",
       "      <td>17.05</td>\n",
       "      <td>192.4</td>\n",
       "      <td>98</td>\n",
       "      <td>8.66</td>\n",
       "      <td>12.3</td>\n",
       "      <td>7</td>\n",
       "      <td>3.32</td>\n",
       "      <td>2</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>104</td>\n",
       "      <td>36.89</td>\n",
       "      <td>152.3</td>\n",
       "      <td>83</td>\n",
       "      <td>12.95</td>\n",
       "      <td>134.3</td>\n",
       "      <td>109</td>\n",
       "      <td>6.04</td>\n",
       "      <td>11.8</td>\n",
       "      <td>4</td>\n",
       "      <td>3.19</td>\n",
       "      <td>2</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>24</td>\n",
       "      <td>170.9</td>\n",
       "      <td>71</td>\n",
       "      <td>29.05</td>\n",
       "      <td>201.4</td>\n",
       "      <td>80</td>\n",
       "      <td>17.12</td>\n",
       "      <td>159.0</td>\n",
       "      <td>124</td>\n",
       "      <td>7.15</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.11</td>\n",
       "      <td>2</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  state  account_length      area_code international_plan voice_mail_plan  \\\n",
       "0    AK               1  area_code_408                 no              no   \n",
       "1    AK              36  area_code_408                 no             yes   \n",
       "2    AK              36  area_code_415                yes             yes   \n",
       "3    AK              41  area_code_415                 no              no   \n",
       "4    AK              42  area_code_415                 no              no   \n",
       "5    AK              48  area_code_415                 no             yes   \n",
       "6    AK              50  area_code_408                 no              no   \n",
       "7    AK              51  area_code_510                yes             yes   \n",
       "8    AK              52  area_code_408                 no              no   \n",
       "9    AK              52  area_code_415                 no             yes   \n",
       "\n",
       "   number_vmail_messages  total_day_minutes  total_day_calls  \\\n",
       "0                      0              175.2               74   \n",
       "1                     30              146.3              128   \n",
       "2                     19              171.9               96   \n",
       "3                      0              159.3               66   \n",
       "4                      0              171.0              129   \n",
       "5                     37              211.7              115   \n",
       "6                      0              183.6              107   \n",
       "7                     12              135.8               60   \n",
       "8                      0              217.0              104   \n",
       "9                     24              170.9               71   \n",
       "\n",
       "   total_day_charge  total_eve_minutes  total_eve_calls  total_eve_charge  \\\n",
       "0             29.78              151.7               79             12.89   \n",
       "1             24.87              162.5               80             13.81   \n",
       "2             29.22              198.4              111             16.86   \n",
       "3             27.08              125.9               75             10.70   \n",
       "4             29.07              183.9               96             15.63   \n",
       "5             35.99              159.9               84             13.59   \n",
       "6             31.21               58.6              118              4.98   \n",
       "7             23.09              200.6              134             17.05   \n",
       "8             36.89              152.3               83             12.95   \n",
       "9             29.05              201.4               80             17.12   \n",
       "\n",
       "   total_night_minutes  total_night_calls  total_night_charge  \\\n",
       "0                230.5                109               10.37   \n",
       "1                129.3                109                5.82   \n",
       "2                321.7                 76               14.48   \n",
       "3                261.9                 76               11.79   \n",
       "4                130.2                 90                5.86   \n",
       "5                144.1                 80                6.48   \n",
       "6                202.6                 99                9.12   \n",
       "7                192.4                 98                8.66   \n",
       "8                134.3                109                6.04   \n",
       "9                159.0                124                7.15   \n",
       "\n",
       "   total_intl_minutes  total_intl_calls  total_intl_charge  \\\n",
       "0                 5.3                 3               1.43   \n",
       "1                14.5                 6               3.92   \n",
       "2                10.5                 1               2.84   \n",
       "3                11.1                 5               3.00   \n",
       "4                 4.6                 6               1.24   \n",
       "5                12.2                 1               3.29   \n",
       "6                 8.7                 3               2.35   \n",
       "7                12.3                 7               3.32   \n",
       "8                11.8                 4               3.19   \n",
       "9                 4.1                 5               1.11   \n",
       "\n",
       "   number_customer_service_calls churn  \n",
       "0                              1    no  \n",
       "1                              0    no  \n",
       "2                              1   yes  \n",
       "3                              1    no  \n",
       "4                              0    no  \n",
       "5                              1    no  \n",
       "6                              1    no  \n",
       "7                              2    no  \n",
       "8                              2    no  \n",
       "9                              2    no  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import data\n",
    "df = pd.read_csv(\"data/Churn_Calls.csv\", sep=',')\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['state', 'account_length', 'area_code', 'international_plan',\n",
      "       'voice_mail_plan', 'number_vmail_messages', 'total_day_minutes',\n",
      "       'total_day_calls', 'total_day_charge', 'total_eve_minutes',\n",
      "       'total_eve_calls', 'total_eve_charge', 'total_night_minutes',\n",
      "       'total_night_calls', 'total_night_charge', 'total_intl_minutes',\n",
      "       'total_intl_calls', 'total_intl_charge',\n",
      "       'number_customer_service_calls', 'churn'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# See each collum name\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 20)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Target\n",
    "In this step I took the target variable and moved it to the first collum. I aslo made a reference to it called targetName. This just helps me with some below steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churn</th>\n",
       "      <th>state</th>\n",
       "      <th>account_length</th>\n",
       "      <th>area_code</th>\n",
       "      <th>international_plan</th>\n",
       "      <th>voice_mail_plan</th>\n",
       "      <th>number_vmail_messages</th>\n",
       "      <th>total_day_minutes</th>\n",
       "      <th>total_day_calls</th>\n",
       "      <th>total_day_charge</th>\n",
       "      <th>total_eve_minutes</th>\n",
       "      <th>total_eve_calls</th>\n",
       "      <th>total_eve_charge</th>\n",
       "      <th>total_night_minutes</th>\n",
       "      <th>total_night_calls</th>\n",
       "      <th>total_night_charge</th>\n",
       "      <th>total_intl_minutes</th>\n",
       "      <th>total_intl_calls</th>\n",
       "      <th>total_intl_charge</th>\n",
       "      <th>number_customer_service_calls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>1</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>175.2</td>\n",
       "      <td>74</td>\n",
       "      <td>29.78</td>\n",
       "      <td>151.7</td>\n",
       "      <td>79</td>\n",
       "      <td>12.89</td>\n",
       "      <td>230.5</td>\n",
       "      <td>109</td>\n",
       "      <td>10.37</td>\n",
       "      <td>5.3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.43</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>30</td>\n",
       "      <td>146.3</td>\n",
       "      <td>128</td>\n",
       "      <td>24.87</td>\n",
       "      <td>162.5</td>\n",
       "      <td>80</td>\n",
       "      <td>13.81</td>\n",
       "      <td>129.3</td>\n",
       "      <td>109</td>\n",
       "      <td>5.82</td>\n",
       "      <td>14.5</td>\n",
       "      <td>6</td>\n",
       "      <td>3.92</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yes</td>\n",
       "      <td>AK</td>\n",
       "      <td>36</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>19</td>\n",
       "      <td>171.9</td>\n",
       "      <td>96</td>\n",
       "      <td>29.22</td>\n",
       "      <td>198.4</td>\n",
       "      <td>111</td>\n",
       "      <td>16.86</td>\n",
       "      <td>321.7</td>\n",
       "      <td>76</td>\n",
       "      <td>14.48</td>\n",
       "      <td>10.5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.84</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>41</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>159.3</td>\n",
       "      <td>66</td>\n",
       "      <td>27.08</td>\n",
       "      <td>125.9</td>\n",
       "      <td>75</td>\n",
       "      <td>10.70</td>\n",
       "      <td>261.9</td>\n",
       "      <td>76</td>\n",
       "      <td>11.79</td>\n",
       "      <td>11.1</td>\n",
       "      <td>5</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>42</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>129</td>\n",
       "      <td>29.07</td>\n",
       "      <td>183.9</td>\n",
       "      <td>96</td>\n",
       "      <td>15.63</td>\n",
       "      <td>130.2</td>\n",
       "      <td>90</td>\n",
       "      <td>5.86</td>\n",
       "      <td>4.6</td>\n",
       "      <td>6</td>\n",
       "      <td>1.24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>48</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>37</td>\n",
       "      <td>211.7</td>\n",
       "      <td>115</td>\n",
       "      <td>35.99</td>\n",
       "      <td>159.9</td>\n",
       "      <td>84</td>\n",
       "      <td>13.59</td>\n",
       "      <td>144.1</td>\n",
       "      <td>80</td>\n",
       "      <td>6.48</td>\n",
       "      <td>12.2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.29</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>50</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>183.6</td>\n",
       "      <td>107</td>\n",
       "      <td>31.21</td>\n",
       "      <td>58.6</td>\n",
       "      <td>118</td>\n",
       "      <td>4.98</td>\n",
       "      <td>202.6</td>\n",
       "      <td>99</td>\n",
       "      <td>9.12</td>\n",
       "      <td>8.7</td>\n",
       "      <td>3</td>\n",
       "      <td>2.35</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>51</td>\n",
       "      <td>area_code_510</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>12</td>\n",
       "      <td>135.8</td>\n",
       "      <td>60</td>\n",
       "      <td>23.09</td>\n",
       "      <td>200.6</td>\n",
       "      <td>134</td>\n",
       "      <td>17.05</td>\n",
       "      <td>192.4</td>\n",
       "      <td>98</td>\n",
       "      <td>8.66</td>\n",
       "      <td>12.3</td>\n",
       "      <td>7</td>\n",
       "      <td>3.32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_408</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>104</td>\n",
       "      <td>36.89</td>\n",
       "      <td>152.3</td>\n",
       "      <td>83</td>\n",
       "      <td>12.95</td>\n",
       "      <td>134.3</td>\n",
       "      <td>109</td>\n",
       "      <td>6.04</td>\n",
       "      <td>11.8</td>\n",
       "      <td>4</td>\n",
       "      <td>3.19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>no</td>\n",
       "      <td>AK</td>\n",
       "      <td>52</td>\n",
       "      <td>area_code_415</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>24</td>\n",
       "      <td>170.9</td>\n",
       "      <td>71</td>\n",
       "      <td>29.05</td>\n",
       "      <td>201.4</td>\n",
       "      <td>80</td>\n",
       "      <td>17.12</td>\n",
       "      <td>159.0</td>\n",
       "      <td>124</td>\n",
       "      <td>7.15</td>\n",
       "      <td>4.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  churn state  account_length      area_code international_plan  \\\n",
       "0    no    AK               1  area_code_408                 no   \n",
       "1    no    AK              36  area_code_408                 no   \n",
       "2   yes    AK              36  area_code_415                yes   \n",
       "3    no    AK              41  area_code_415                 no   \n",
       "4    no    AK              42  area_code_415                 no   \n",
       "5    no    AK              48  area_code_415                 no   \n",
       "6    no    AK              50  area_code_408                 no   \n",
       "7    no    AK              51  area_code_510                yes   \n",
       "8    no    AK              52  area_code_408                 no   \n",
       "9    no    AK              52  area_code_415                 no   \n",
       "\n",
       "  voice_mail_plan  number_vmail_messages  total_day_minutes  total_day_calls  \\\n",
       "0              no                      0              175.2               74   \n",
       "1             yes                     30              146.3              128   \n",
       "2             yes                     19              171.9               96   \n",
       "3              no                      0              159.3               66   \n",
       "4              no                      0              171.0              129   \n",
       "5             yes                     37              211.7              115   \n",
       "6              no                      0              183.6              107   \n",
       "7             yes                     12              135.8               60   \n",
       "8              no                      0              217.0              104   \n",
       "9             yes                     24              170.9               71   \n",
       "\n",
       "   total_day_charge  total_eve_minutes  total_eve_calls  total_eve_charge  \\\n",
       "0             29.78              151.7               79             12.89   \n",
       "1             24.87              162.5               80             13.81   \n",
       "2             29.22              198.4              111             16.86   \n",
       "3             27.08              125.9               75             10.70   \n",
       "4             29.07              183.9               96             15.63   \n",
       "5             35.99              159.9               84             13.59   \n",
       "6             31.21               58.6              118              4.98   \n",
       "7             23.09              200.6              134             17.05   \n",
       "8             36.89              152.3               83             12.95   \n",
       "9             29.05              201.4               80             17.12   \n",
       "\n",
       "   total_night_minutes  total_night_calls  total_night_charge  \\\n",
       "0                230.5                109               10.37   \n",
       "1                129.3                109                5.82   \n",
       "2                321.7                 76               14.48   \n",
       "3                261.9                 76               11.79   \n",
       "4                130.2                 90                5.86   \n",
       "5                144.1                 80                6.48   \n",
       "6                202.6                 99                9.12   \n",
       "7                192.4                 98                8.66   \n",
       "8                134.3                109                6.04   \n",
       "9                159.0                124                7.15   \n",
       "\n",
       "   total_intl_minutes  total_intl_calls  total_intl_charge  \\\n",
       "0                 5.3                 3               1.43   \n",
       "1                14.5                 6               3.92   \n",
       "2                10.5                 1               2.84   \n",
       "3                11.1                 5               3.00   \n",
       "4                 4.6                 6               1.24   \n",
       "5                12.2                 1               3.29   \n",
       "6                 8.7                 3               2.35   \n",
       "7                12.3                 7               3.32   \n",
       "8                11.8                 4               3.19   \n",
       "9                 4.1                 5               1.11   \n",
       "\n",
       "   number_customer_service_calls  \n",
       "0                              1  \n",
       "1                              0  \n",
       "2                              1  \n",
       "3                              1  \n",
       "4                              0  \n",
       "5                              1  \n",
       "6                              1  \n",
       "7                              2  \n",
       "8                              2  \n",
       "9                              2  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# designate target variable name\n",
    "targetName = 'churn'\n",
    "# move target variable into first column\n",
    "targetSeries = df[targetName]\n",
    "del df[targetName]\n",
    "df.insert(0, targetName, targetSeries)\n",
    "expected=targetName\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#EDA\n",
    "Just a touch of EDA. This is the distribution of the target. As you can see, the datset is imbalanced and the target class of interest \"yes\" is in the minority (a common occurance in classification)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x10c24b2b0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAERCAYAAABy/XBZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADtZJREFUeJzt3X+s3XV9x/HnSxBctoUfUg1ryUq0W8Rs/liDJLiFgIGiZGULGMwyO0PWZGEbJss23C+cSoJbNozZJGPSWc0y7JgR4khIh8JijGIRZQPCWhGkQ6WsgL9xxff+OJ+6a73tvRduz4H7fj6Sm3O+n+/nnPs5yW2f93vO99yTqkKS1M/zZr0ASdJsGABJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0dOesFHMoJJ5xQa9eunfUyJOk55Y477ni0qlYtNO9ZHYC1a9eyY8eOWS9Dkp5Tkjy4mHk+BSRJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqaln9RvBnivWXvavs17CivLAlW+Y9RKkFjwCkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1tegAJDkiyZ1JPja2T07ymSQ7k3w4yVFj/OixvWvsXzvnPt42xu9Lcs5yPxhJ0uIt5QjgUuDeOdvvBq6qqnXAY8DFY/xi4LGqeilw1ZhHklOAi4CXAxuA9yU54pktX5L0dC0qAEnWAG8A3j+2A5wJXD+mbAXOH9c3jm3G/rPG/I3AdVX1ZFV9CdgFnLocD0KStHSLPQJ4D/AHwPfH9guBx6tq39jeDawe11cDDwGM/U+M+T8Yn+c2kqQpWzAASc4DHqmqO+YOzzO1Fth3qNvM/X6bk+xIsmPPnj0LLU+S9DQt5gjgdOCXkzwAXMfkqZ/3AMcm2f95AmuAh8f13cBJAGP/McDeuePz3OYHquqaqlpfVetXrVq15AckSVqcBQNQVW+rqjVVtZbJi7gfr6pfAz4BXDCmbQJuGNdvHNuM/R+vqhrjF42zhE4G1gG3L9sjkSQtyTP5RLA/BK5L8i7gTuDaMX4t8KEku5j85n8RQFXdnWQbcA+wD7ikqp56Bt9fkvQMLCkAVXUrcOu4fj/znMVTVd8FLjzI7a8ArljqIiVJy893AktSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0tGIAkL0hye5IvJLk7yZ+P8ZOTfCbJziQfTnLUGD96bO8a+9fOua+3jfH7kpxzuB6UJGlhizkCeBI4s6peAbwS2JDkNODdwFVVtQ54DLh4zL8YeKyqXgpcNeaR5BTgIuDlwAbgfUmOWM4HI0lavAUDUBPfHJvPH18FnAlcP8a3AueP6xvHNmP/WUkyxq+rqier6kvALuDUZXkUkqQlW9RrAEmOSPJ54BFgO/BF4PGq2jem7AZWj+urgYcAxv4ngBfOHZ/nNpKkKVtUAKrqqap6JbCGyW/tL5tv2rjMQfYdbPyHJNmcZEeSHXv27FnM8iRJT8OSzgKqqseBW4HTgGOTHDl2rQEeHtd3AycBjP3HAHvnjs9zm7nf45qqWl9V61etWrWU5UmSlmAxZwGtSnLsuP5jwOuAe4FPABeMaZuAG8b1G8c2Y//Hq6rG+EXjLKGTgXXA7cv1QCRJS3PkwlM4Edg6zth5HrCtqj6W5B7guiTvAu4Erh3zrwU+lGQXk9/8LwKoqruTbAPuAfYBl1TVU8v7cCRJi7VgAKrqLuBV84zfzzxn8VTVd4ELD3JfVwBXLH2ZkqTl5juBJakpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDW1YACSnJTkE0nuTXJ3kkvH+PFJtifZOS6PG+NJ8t4ku5LcleTVc+5r05i/M8mmw/ewJEkLWcwRwD7g96rqZcBpwCVJTgEuA26pqnXALWMb4Fxg3fjaDFwNk2AAlwOvAU4FLt8fDUnS9C0YgKr6SlV9blz/BnAvsBrYCGwd07YC54/rG4EP1sSngWOTnAicA2yvqr1V9RiwHdiwrI9GkrRoS3oNIMla4FXAZ4AXV9VXYBIJ4EVj2mrgoTk32z3GDjZ+4PfYnGRHkh179uxZyvIkSUuw6AAk+QngX4C3VtXXDzV1nrE6xPgPD1RdU1Xrq2r9qlWrFrs8SdISLSoASZ7P5D//f6yqj4zhr42ndhiXj4zx3cBJc26+Bnj4EOOSpBlYzFlAAa4F7q2qv56z60Zg/5k8m4Ab5oy/eZwNdBrwxHiK6Gbg7CTHjRd/zx5jkqQZOHIRc04Hfh34jySfH2N/BFwJbEtyMfBl4MKx7ybg9cAu4NvAWwCqam+SdwKfHfPeUVV7l+VRSJKWbMEAVNUnmf/5e4Cz5plfwCUHua8twJalLFCSdHj4TmBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKmpBQOQZEuSR5L855yx45NsT7JzXB43xpPkvUl2Jbkryavn3GbTmL8zyabD83AkSYu1mCOADwAbDhi7DLilqtYBt4xtgHOBdeNrM3A1TIIBXA68BjgVuHx/NCRJs7FgAKrq34G9BwxvBLaO61uB8+eMf7AmPg0cm+RE4Bxge1XtrarHgO38aFQkSVP0dF8DeHFVfQVgXL5ojK8GHpozb/cYO9j4j0iyOcmOJDv27NnzNJcnSVrIcr8InHnG6hDjPzpYdU1Vra+q9atWrVrWxUmS/t/TDcDXxlM7jMtHxvhu4KQ589YADx9iXJI0I083ADcC+8/k2QTcMGf8zeNsoNOAJ8ZTRDcDZyc5brz4e/YYkyTNyJELTUjyT8AZwAlJdjM5m+dKYFuSi4EvAxeO6TcBrwd2Ad8G3gJQVXuTvBP47Jj3jqo68IVlSdIULRiAqnrTQXadNc/cAi45yP1sAbYsaXWSpMPGdwJLUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWpqwTeCSXqOe/sxs17ByvH2J2a9gmXlEYAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpqaegCSbEhyX5JdSS6b9veXJE1MNQBJjgD+FjgXOAV4U5JTprkGSdLEtI8ATgV2VdX9VfU94Dpg45TXIEkCjpzy91sNPDRnezfwmoNNvu+++zjjjDMO95qesa/e/z+zXsKKcsan/3LWS1hZHvjWrFewctx6xqxXsKymHYDMM1Y/NCHZDGwem9+87bbb7jvsq+rjBODRWS9iIbc9tPAcrTjPiZ9NHrxt1itYrJ9ezKRpB2A3cNKc7TXAw3MnVNU1wDXTXFQXSXZU1fpZr0M6kD+bszHt1wA+C6xLcnKSo4CLgBunvAZJElM+AqiqfUl+G7gZOALYUlV3T3MNkqSJaT8FRFXdBNw07e8rwKfW9Ozlz+YMpKoWniVJWnH8UxCS1JQBkKSmDIAkNWUAJM1EkguT/OS4/idJPpLk1bNeVycGYIVLckySq5LsGF9/leSYWa9LAv60qr6R5LXAOcBW4OoZr6kVA7DybQG+DrxxfH0d+IeZrkiaeGpcvgG4uqpuAI6a4Xra8TTQFS7J56vqlQuNSdOW5GPAfwOvA34B+A5we1W9YqYLa8QjgJXvO+MQG4AkpzP5hybN2huZ/FWADVX1OHA88PuzXVIvU38nsKbut4Ctc573fwzYNMP1SABU1beTPAK8FtgJ7BuXmhKfAlrhkhwNXAC8BDgWeAKoqnrHTBem9pJcDqwHfraqfibJTwH/XFWnz3hpbXgEsPLdADwOfI7J863Ss8WvAK9i8rNJVT28/7RQTYcBWPnWVNWGWS9Cmsf3qqqSFECSH5/1grrxReCV71NJfm7Wi5DmsS3J3wHHJvlN4N+Av5/xmlrxNYAVLsk9wEuBLwFPMvlYzqqqn5/pwtRekt8BvgqcyuTn8uaq2j7bVfXiU0Ar37mzXoB0EC8GLmXyGsAWJkcAmiKPACTNTJIAZwNvYXJG0Dbg2qr64kwX1oSvAUiamZr8BvrV8bUPOA64PslfzHRhTXgEIGkmkvwukzclPgq8H/hoVf1vkucBO6vqJTNdYAO+BiBpVk4AfrWqHpw7WFXfT3LejNbUikcAktSUrwFIUlMGQJKaMgDSPJJ8IMkFs16HdDgZAOkwSHLErNcgLcQASECSNye5K8kXknxoDP9Skk8luX//0UCSM8YnWe2/3d8k+Y1x/YEkf5bkk8CFSW5N8u4ktyf5ryS/OPUHJh2CAVB7SV4O/DFw5vg4wkvHrhOZfFjJecCVi7y771bVa6vqurF9ZFWdCrwVuHwZly09YwZAgjOB66vqUYCq2jvGP1pV36+qe5j83ZrF+PAB2x8Zl3cAa5/pQqXlZACk8RdS5xl/8oA5MPlzBXP/3bzggNt86yD38RS+8VLPMgZAgluANyZ5IUCS4w8x90HglCRHj89ZPmsaC5QOB38jUXtVdXeSK4DbkjwF3HmIuQ8l2QbcxeQDzA86V3q2809BSFJTPgUkSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKmp/wPaHoDFgbdKTwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gb = df.groupby(targetName)\n",
    "targetEDA=gb[targetName].aggregate(len)\n",
    "plt.figure()\n",
    "targetEDA.plot(kind='bar', grid=False)\n",
    "plt.axhline(0, color='k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing\n",
    "The below two steps are for preprocessing. The first cell changes the yes/no of the target to numeric. I needed to do this as some models require the target to be numeric. The second cell takes all the category features and creates dummies with them. This is stock code I have used for long time (and I did not write it). It is nice because it will take any dataframe of any size and handle categorial features. I do not have to change a single line in it. It can be used generically on bascially any dataframe. Saves a lot of time of coding each feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "le_dep = preprocessing.LabelEncoder()\n",
    "#to convert into numbers\n",
    "df['churn'] = le_dep.fit_transform(df['churn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# perform data transformation\n",
    "for col in df.columns[1:]:\n",
    "\tattName = col\n",
    "\tdType = df[col].dtype\n",
    "\tmissing = pd.isnull(df[col]).any()\n",
    "\tuniqueCount = len(df[attName].value_counts(normalize=False))\n",
    "\t# discretize (create dummies)\n",
    "\tif dType == object:\n",
    "\t\tdf = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)\n",
    "\t\tdel df[attName]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# After Tranformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 74)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churn</th>\n",
       "      <th>account_length</th>\n",
       "      <th>number_vmail_messages</th>\n",
       "      <th>total_day_minutes</th>\n",
       "      <th>total_day_calls</th>\n",
       "      <th>total_day_charge</th>\n",
       "      <th>total_eve_minutes</th>\n",
       "      <th>total_eve_calls</th>\n",
       "      <th>total_eve_charge</th>\n",
       "      <th>total_night_minutes</th>\n",
       "      <th>...</th>\n",
       "      <th>state_WI</th>\n",
       "      <th>state_WV</th>\n",
       "      <th>state_WY</th>\n",
       "      <th>area_code_area_code_408</th>\n",
       "      <th>area_code_area_code_415</th>\n",
       "      <th>area_code_area_code_510</th>\n",
       "      <th>international_plan_no</th>\n",
       "      <th>international_plan_yes</th>\n",
       "      <th>voice_mail_plan_no</th>\n",
       "      <th>voice_mail_plan_yes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>175.2</td>\n",
       "      <td>74</td>\n",
       "      <td>29.78</td>\n",
       "      <td>151.7</td>\n",
       "      <td>79</td>\n",
       "      <td>12.89</td>\n",
       "      <td>230.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>30</td>\n",
       "      <td>146.3</td>\n",
       "      <td>128</td>\n",
       "      <td>24.87</td>\n",
       "      <td>162.5</td>\n",
       "      <td>80</td>\n",
       "      <td>13.81</td>\n",
       "      <td>129.3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>19</td>\n",
       "      <td>171.9</td>\n",
       "      <td>96</td>\n",
       "      <td>29.22</td>\n",
       "      <td>198.4</td>\n",
       "      <td>111</td>\n",
       "      <td>16.86</td>\n",
       "      <td>321.7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>159.3</td>\n",
       "      <td>66</td>\n",
       "      <td>27.08</td>\n",
       "      <td>125.9</td>\n",
       "      <td>75</td>\n",
       "      <td>10.70</td>\n",
       "      <td>261.9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>129</td>\n",
       "      <td>29.07</td>\n",
       "      <td>183.9</td>\n",
       "      <td>96</td>\n",
       "      <td>15.63</td>\n",
       "      <td>130.2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   churn  account_length  number_vmail_messages  total_day_minutes  \\\n",
       "0      0               1                      0              175.2   \n",
       "1      0              36                     30              146.3   \n",
       "2      1              36                     19              171.9   \n",
       "3      0              41                      0              159.3   \n",
       "4      0              42                      0              171.0   \n",
       "\n",
       "   total_day_calls  total_day_charge  total_eve_minutes  total_eve_calls  \\\n",
       "0               74             29.78              151.7               79   \n",
       "1              128             24.87              162.5               80   \n",
       "2               96             29.22              198.4              111   \n",
       "3               66             27.08              125.9               75   \n",
       "4              129             29.07              183.9               96   \n",
       "\n",
       "   total_eve_charge  total_night_minutes         ...           state_WI  \\\n",
       "0             12.89                230.5         ...                  0   \n",
       "1             13.81                129.3         ...                  0   \n",
       "2             16.86                321.7         ...                  0   \n",
       "3             10.70                261.9         ...                  0   \n",
       "4             15.63                130.2         ...                  0   \n",
       "\n",
       "   state_WV  state_WY  area_code_area_code_408  area_code_area_code_415  \\\n",
       "0         0         0                        1                        0   \n",
       "1         0         0                        1                        0   \n",
       "2         0         0                        0                        1   \n",
       "3         0         0                        0                        1   \n",
       "4         0         0                        0                        1   \n",
       "\n",
       "   area_code_area_code_510  international_plan_no  international_plan_yes  \\\n",
       "0                        0                      1                       0   \n",
       "1                        0                      1                       0   \n",
       "2                        0                      0                       1   \n",
       "3                        0                      1                       0   \n",
       "4                        0                      1                       0   \n",
       "\n",
       "   voice_mail_plan_no  voice_mail_plan_yes  \n",
       "0                   1                    0  \n",
       "1                   0                    1  \n",
       "2                   0                    1  \n",
       "3                   1                    0  \n",
       "4                   1                    0  \n",
       "\n",
       "[5 rows x 74 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test/Train\n",
    "I split the data into a 60/40 train test. The features are stored in \"features_train\" and \"features_test\". The targets are in \"target_train\" and \"target_test\". I used a biggest test when I have an imbalanced set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split dataset into testing and training\n",
    "features_train, features_test, target_train, target_test = train_test_split(\n",
    "    df.iloc[:,1:].values, df.iloc[:,0].values, test_size=0.40, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just a view of the size of each test/train set.\n",
    "Note there are now 73 features, and the test set is imbalanced (14.6%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 73)\n",
      "(3000, 73)\n",
      "(2000,)\n",
      "(3000,)\n",
      "Percent of Target that is Yes 0.146\n"
     ]
    }
   ],
   "source": [
    "print(features_test.shape)\n",
    "print(features_train.shape)\n",
    "print(target_test.shape)\n",
    "print(target_train.shape)\n",
    "print(\"Percent of Target that is Yes\", target_test.mean())\n",
    "#data.groupby(['col1', 'col2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Models\n",
    "All the models are done in Sci-Kit Learn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "Logistic Accuracy Score 0.861\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "  Fail = no       0.88      0.97      0.92      1708\n",
      " Fail = yes       0.56      0.23      0.32       292\n",
      "\n",
      "avg / total       0.83      0.86      0.83      2000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf_Log=LogisticRegression()\n",
    "print(clf_Log)\n",
    "clf_Log = clf_Log.fit(features_train, target_train)\n",
    "#DT test model\n",
    "target_predicted_Log = clf_Log.predict(features_test)\n",
    "print(\"Logistic Accuracy Score\", accuracy_score(target_test, target_predicted_Log))\n",
    "# print classification report\n",
    "target_names = [\"Fail = no\", \"Fail = yes\"]\n",
    "print(classification_report(target_test, target_predicted_Log, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree\n",
    "I created a decision tree from the data. The accurancy of the model was 921%, while the test data classified at 92%. However notice that the \"yes\" class (the class I am interested in) only properly classified at 74% (specificity) and .71 (recall). That is so-so. Again, not uncommon with imbalanced data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
      "            max_features=None, max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
      "            splitter='best')\n",
      "DT Accuracy Score 0.924\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "  Fail = no       0.95      0.96      0.96      1708\n",
      " Fail = yes       0.74      0.73      0.74       292\n",
      "\n",
      "avg / total       0.92      0.92      0.92      2000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Decision Tree train model\n",
    "from sklearn import tree \n",
    "clf = tree.DecisionTreeClassifier()\n",
    "print(clf)\n",
    "clf = clf.fit(features_train, target_train)\n",
    "#DT test model\n",
    "target_predicted_dt = clf.predict(features_test)\n",
    "print(\"DT Accuracy Score\", accuracy_score(target_test, target_predicted_dt))\n",
    "# print classification report\n",
    "target_names = [\"Fail = no\", \"Fail = yes\"]\n",
    "print(classification_report(target_test, target_predicted_dt, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross Validation of Decision Tree\n",
    "I cross validated with 10 repeats. You can see the OOB score for each repeat and the mean. The mean is .92, which is quite close to the orginal model. I am not going to worry about over fitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross Validation Score for each K [0.9269103  0.90365449 0.91694352 0.93023256 0.91362126 0.94314381\n",
      " 0.92307692 0.92976589 0.93311037 0.92307692]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9243536039289324"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#verify DT with Cross Validation\n",
    "scores = cross_val_score(clf, features_train, target_train, cv=10)\n",
    "print(\"Cross Validation Score for each K\",scores)\n",
    "scores.mean()                             "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visual of Confusion Matrix for Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1635   73]\n",
      " [  79  213]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARIAAAD0CAYAAABXXakIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGzRJREFUeJzt3XuYXFWZ7/Hvj4Rb5E4AIYBBDDiYBxhERDwiihMuw4DOIw6IgsiZDMhwRhkdQTjijdEZ5xFBkZlgEFEHElGew4EcY8yIiA8XCXKLQIgwYDDcISK3JN3v+WOtSoqmq2pX7ars7qrfh2c/3bX2qtprF+m31lp71X4VEZiZlbFe1Q0ws/HPgcTMSnMgMbPSHEjMrDQHEjMrzYHEzEpzIDGz0hxIKiZpY0n/V9IKST8s8TrHSfppN9tWFUnvkHRf1e2w4uQFacVI+iBwOvBG4DngduDciLih5Ot+GDgNOCAiVpdu6BgnKYBpEbG06rZY90ysugHjgaTTgTOAk4H5wErgUOAooFQgAV4HLBmEIFKEpImD9F4c8q7XxFNPDxWqu+jOl+dHxKE9blJnIsJbkw3YHPgTcHSTOhsCXwf+kLevAxvmfQcBy4B/BB4HlgMn5n2fJwWlVfkYJwGfA75f99pTgQAm5scfAR4g9YoeBI6rK7+h7nkHAL8GVuSfB9Ttuw74IvCr/Do/BSY3OLda+/+prv3vBQ4HlgBPA5+pq78fcCPwbK77TWCDvO/6fC7P5/P9m7rX/zTwKPC9Wll+zq75GPvkxzsATwIHVf1voxvbPntuGKuW71poA26tur2NNs+RtPY2YCPgqiZ1zgL2B/YG9iL9MZ1dt/+1pIA0hRQsLpS0ZUScA/wzMCciNomI2c0aIuk1wAXAYRGxKSlY3D5Kva2Aa3PdrYGvAddK2rqu2geBE4FtgQ2ATzY59GtJ78EU4LPAxcCHgDcD7wA+K+n1ue4Q8AlgMum9Oxj4GEBEHJjr7JXPd07d629F6p3NrD9wRPyOFGR+IGkS8B3g0oi4rkl7x5FgKIYLbUVIukTS45LuHlF+mqT7JC2W9K915WdKWpr3HVJXfmguWyrpjFbHdSBpbWvgyWje3T4O+EJEPB4RT5B6Gh+u278q718VEfNIn8a7d9ieYWC6pI0jYnlELB6lzl8C90fE9yJidURcDtwL/FVdne9ExJKIeBGYSwqCjawizQetAq4gBYnzI+K5fPzFwJ4AEbEoIm7Kx/1v4D+AdxY4p3Mi4uXcnleIiIuB+4Gbge1JgbsvBDBMFNoKupQ07F5D0rtIw/A9I+JNwL/l8j2AY4A35ed8S9IESROAC4HDgD2AY3PdhhxIWnsKmCyp2XzSDsBDdY8fymVrXmNEIHoB2KTdhkTE86ThwMnAcknXSnpjgfbU2jSl7vGjbbTnqYioDeRrf+iP1e1/sfZ8SbtJukbSo5L+SOpxTW7y2gBPRMRLLepcDEwHvhERL7eoO64MF/yviIi4njQUrHcK8JXa+xYRj+fyo4ArcgB/EFhK6k3vByyNiAciYiXpw+OoZsd1IGntRuAl0rxAI38gdctrds5lnXgemFT3+LX1OyNifkT8BemT+V7SH1ir9tTa9EiHbWrHRaR2TYuIzYDPAGrxnKYft5I2Ic07zQY+l4dufSEIhqLYRvpAu7Vum9nq9bPdgHdIulnSLyS9JZdPAX5fV29ZLmtU3pADSQsRsYI0L3ChpPdKmiRpfUlnS3pa0lLSp/3ZkraRNDnX/36Hh7wdOFDSzpI2B86s7ZC0naQj81zJy6Qh0mhT/vOA3SR9UNJESX9D6qJe02Gb2rEp8EfgT7m3dMqI/Y8Br3/Vs15ty7qx/vnAooj4n6S5n3/vZoOr1sbQ5smI2Ldum1XwEBOBLUnzeJ8C5koSowf4aFLekANJARHxNdIakrOBJ0jR+kzSxOAepK77A8CdwF3AbcCXOjzWAmBOfq1FvPKPfz3S1Z8/kLqv7yRPZI54jaeAI3Ldp0hXXI6IiCc7aVObPkmayH2O1FuaM2L/54DvSnpW0geavM7zpHH7pvnnybn8dGAfScd1s9FVCWCIKLSVsAz4cSS3kOakJufynerq7Uj6t9WovCEvSOuApLcBn4uIQ/LjMwEi4suVNqzPSJoKXBMR0ytuSs/stdcGMX9eqymkZPsdly+KiH1b1Rv5vkk6GdghIj4raTdgIWmouwfwn6Q5kR1y+TRSj2QJ6YrbI6TlAx9sMLEPeEFap0YbQ761orbYOFdsGrUYSZeT1uFMlrQMOAe4BLgkDxNXAidE6kEsljQX+C2wGji1Nqku6e9Jiy8nAJc0CyLgQNKptseQZqOJ8sOWV75exLENdn2oQf1zgXNHKZ9HmmsrxIGkM22PIc1GFTDUBx9BnmztzK+BaZJ2kbQBaVHP1RW3ycahtCCt2DaWOZB0IC8uq40h7wHmthpDWnvyWP9GYHdJyySdVHWbekMMFdzGMg9tOtTuGNLa02Ss31cCGO6DoY0DiVmFAljZBwMDBxKzig3H2B62FOFAYlahtLLVgcTMSgjEUB8Mbcb/GVSojW9fWocG4T0eDhXaxjIHknL6/h/5GNDX73FtaOPLv2ZWghiK8f95PqYCyeStJsTUndavuhmF7TxlIvvutdG4WgWw5M5JrSuNIRsxic201bh6j1/ieVbGy4W6EAGsYkKPW9R7YyqQTN1pfW6Zv1PritaxQ3ZodmtW64abY2HhuhHukZhZFwyP8fmPIhxIzCqUJlvdIzGzUjy0MbOS0m0EHEjMrIRArAxftTGzkob7YGgz/s/AbByrTbYW2YpolPs37/ukpMi5l1ByQc7ve6ekferqniDp/ryd0Oq4DiRmFQrEUBTbCrqUEbl/ASTtBPwF8HBd8WGk9BPTSF9FuCjX3Yp09/m3klJVnCNpy2YHdSAxq9gw6xXaimiQ+xfgPFKitPpVwkcBl+XEWTcBW0jaHjgEWBART0fEM8ACRglO9TxHYlahCHp++VfSkcAjEXFHytS5Rtdy/zqQmFVK7axsnSzp1rrHs1rl/5U0CTgLmDHqwV+to9y/DiRmFQpgZRT+M3yySMrOEXYFdgFqvZEdgdsk7Ufz3L8HjSi/rtlBPEdiVqGg2E2NOr2xUUTcFRHbRsTUiJhKChL7RMSjpFxMx+erN/sDKyJiOSnNygxJW+ZJ1hm5rCH3SMwq1s3v2oyW+zciZjeoPg84HFgKvACcCBART0v6IikRHMAXImK0Cdw1HEjMKpTy2nQvkLTKB5R7JbXfAzi1Qb1LSMnHC3EgMavU2L+NYhEOJGYV6naPpCoOJGYVc4/EzEqJEKuGx/+f4fg/A7NxLN2PxD0SMyvFd0gzs5LSZKt7JGZWkm/+bGal1JbIj3cOJGYV882fzayUCFg17EBiZiWkoY0DiZmV5JWtZlaKL/+aWRd4aGNmXeAl8mZWSrqLvAOJmZUQiNXDzv1rZiV5aGNmpfTLVZvxP11sNs4Nx3qFtiJGSyIu6auS7s2Jwq+StEXdvjNzEvH7JB1SV35oLlsq6YxWx3UgMatSwZw2bfRaLuXVeXoXANMjYk9gCXAmgKQ9gGOAN+XnfEvSBEkTgAtJScb3AI7NdRtyIDGrUO0OaUW2Qq83ShLxiPhpRKzOD28iZc6DlET8ioh4OSIeJOW32S9vSyPigYhYCVyR6zbkORKzirXR22g79+8oPgrMyb9PIQWWmvpk4SOTiL+12Ys6kJhVKIDVxb/920nu3zUknQWsBn5QK2rQpNEa1DSJeE+HNu1O2JgNml7n/q2RdAJwBHBczrAHzZOIj1beUM8CSScTNmaDqJtzJKORdCjwaeDIiHihbtfVwDGSNpS0CzANuIWU83eapF0kbUCakL262TF6ObRZM2EDIKk2YfPbHh7TbHyJ7q4jGS2JOOkqzYbAAkkAN0XEyRGxWNJc0t/kauDUiBjKr/P3wHxgAnBJRCxudtxeBpIpFJiwkTQTmAmw8xRP2dhg6faCtAZJxGc3qX8ucO4o5fOAeUWP28s5kkYTOa8siJgVEftGxL7bbD3+v3Ng1q51MUfSa73sArQ9YWM2aAIx1Af3bO3lGbQ9YWM2iHo92bou9KxHEhGr252wMRs00eXJ1qr0dHaz3Qkbs0EUDiRmVs7Yn0gtwoHErGLukZhZKf1yYyMHErMq+ebPZlZW4KGNmZXmyVYz64JoeqeP8cGBxKxiHtqYWSkRDiRm1gWeIzGz0oaHHUjMrIRAHtqYWXl9cNHGgcSsUn0y2Tr+b81kNt5Fwa2ABrl/t5K0QNL9+eeWuVySLsjpYu6UtE/dc07I9e/PqSyaahhIJG3WbCt2WmbWSoQKbQVdyqtz/54BLIyIacDC/BhSqphpeZsJXAQp8JDuPv9WUjaIc2rBp5FmQ5vFpDhYfwa1xwHs3OqMzKy1bq5sjYjrJU0dUXwUKUUFwHeB60h5bo4CLssJs26StIWk7XPdBRHxNICkBaTgdHmj4zYMJBGxU6N9ZtYdERC9v/nzdhGxPB0vlkvaNpePljJmSpPyhgqdgaRjJH0m/76jpDcXa7+ZtZJWt7beyEnE67aZJQ/dKGVMoVQy9VpetZH0TWB94EDgn4EXgH8H3tKymWbWWvGhTadJxB+TtH3ujWwPPJ7Lm+X+PWhE+XXNDlCkR3JARPwd8BJAHjdtUKT1ZtZKsYnWkpeIrwZqV15OAP5PXfnx+erN/sCKPASaD8yQtGWeZJ2Ryxoqso5klaT1yHFT0tbAcNunYmaj6+Jka4Pcv18B5ko6CXgYODpXnwccDiwljTROhNRZkPRFUm4qgC/UJl4bKRJILgR+BGwj6fPAB4DPFz81M2uoywvSGuT+BTh4lLoBnNrgdS4BLil63JaBJCIuk7QIeE8uOjoi7m72HDNrQx+skS+6RH4CsIp0yl4Na9ZNg7BEXtJZpIUoO5Bmb/9T0pm9bpjZwOjiEvmqFOmRfAh4c0S8ACDpXGAR8OVeNsxsIAR90SMpEkgeGlFvIvBAb5pjNnj6+ubPks4jxcsXgMWS5ufHM4Ab1k3zzAZAPwcSoHZlZjFwbV35Tb1rjtkA6uehTUTMXpcNMRtIAeqD5Z1FvmuzK3AusAewUa08InbrYbvMBoT6okdSZE3IpcB3SN8IPAyYC1zRwzaZDZY+uPxbJJBMioj5ABHxu4g4G3hXb5tlNkD6IJAUufz7siQBv5N0MvAIsG2L55hZUWM8SBRRJJB8AtgE+F+kuZLNgY/2slFmA2NQFqRFxM351+eAD/e2OWaDR/3cI5F0FU06XRHx1z1pkdmg6edAAnxznbUiW3LnJA7ZYe91fdiBMvF1vqd3r+kP67dXv58DSUQsXJcNMRtYgzBHYmY9NA4u7RbhQGJWtUEKJJI2jIiXe9kYs0HUD3MkRe6Qtp+ku4D78+O9JH2j5y0zGxTdTSL+CUmLJd0t6XJJG0naRdLNOSH4HEkb5Lob5sdL8/6pnZ5CkSXyFwBHAE8BRMQdeIm8WVcof/u3yNbytaQppIWj+0bEdNK9lo8B/gU4LycRfwY4KT/lJOCZiHgDcF6u15EigWS9iHhoRNlQpwc0sxFCxbZiJgIbS5oITAKWA+8Grsz7vwu8N/9+VH5M3n9w/jpM24oEkt9L2g8ISRMkfRxY0snBzGwUxYc2TXP/RsQjwL+RkmAtB1aQ7q/8bESsztXqE4KvSRae968Atu7kFIpMtp5CGt7sDDwG/CyXmVkXtDHZ2jT3b06veRSwC/As8EPSrT9Gqh2x7WThjRT5rs3jpHGWmfVC967avAd4MCKeAJD0Y+AAYAtJE3Ovo5YoHNYmEV+Wh0KbA01TczZS5A5pFzPKqUbEzFGqm1k7oquXfx8G9pc0CXiRlKbzVuDnwPtJNyQbmUT8BODGvP+/chrPthUZ2vys7veNgPeRx1Vm1gVdCiQRcbOkK4HbgNXAb4BZpJu3XyHpS7msdj/m2cD3JC0l9UQ6HnkUGdrMqX8s6XvAgk4PaGav1M2bP0fEOcA5I4ofAPYbpe5LwNHdOG4neXx3AV7XjYObWX8oMkfyDGs7X+uRukBn9LJRZgOlD5bINw0keXHKXqT7tAIMdzoZY2aj6O5ka2WaDm1y0LgqIoby1genbDbG9MFd5IvMkdwiaZ+et8RsUPVBIGl2z9baApb/AfytpN8Bz5NWw0VEOLiYlST6Y2jTbI7kFmAf1n7Bx8y6bQBy/wpSdr111BazwdTnPZJtJJ3eaGdEfK0H7TEbPH0eSCaQMuyN/1tcm41h/T5HsjwivrDOWmI2qPo8kLgnYtZr4+DSbhHNAsnB66wVZgOsr6/aRERHNzgxs/b0+xyJma0LDiRmVsoAzJGYWY+J/riq4UBiVjX3SMysLE+2mll5fXD5t5N7tppZt+Q7pBXZipC0haQrJd0r6R5Jb5O0laQFOYn4gpxICyUX5CTid5a575ADiVnVuntjo/OBn0TEG0m3Sb2HdI/lhTmJ+ELW3nP5MGBa3mYCF3V6Cg4kZhXrVo9E0mbAgeS8NRGxMiKe5ZXJwkcmEb8skptIGfm27+QcHEjMqtalJOLA64EngO9I+o2kb0t6DbBdRCwHyD+3zfXXJBHP6hOMt8WTrWYV61YScdLf8z7AaTnr3vk0Tx3TtSTi7pGYValob6TYn/cyYFlE3JwfX0kKLI/Vhiz55+N19Xeqe359gvG2OJCYVUikb/8W2VqJiEeB30vaPRcdDPyWtcnC4dVJxI/PV2/2B1bUhkDt8tDGrGrdXZB2GvADSRuQcv6eSOowzJV0EvAwa/P9zgMOB5YCL+S6HelZIJF0CXAE8HhETO/VcczGO3Ux71xE3A6MNo/yqvsL5YR3p3bjuL0c2lwKHNrD1zcb/7o7R1KZnvVIIuJ6SVN79fpm/cLftemCfC18JsBGTKq4NWYVcCApLyJmAbMANtNWffCWmrXHPRIzK2cAUnaa2brQBz2Snl21kXQ5cCOwu6Rl+Rq2mdUR3b2NQFV6edXm2F69tllf6eI6kqp4aGNWsbHe2yjCgcSsSuNgsVkRDiRmFfNVGzMrzYHEzMoJPNlqZuV5stXMynMgMbMyagvSxjsHErMqRXiOxMzK81UbMyutH4Y2vou8WZUCGI5iW0GSJuQEWdfkx7tIujnn/p2TbwyNpA3z46V5/9ROT8OBxKxq3b9n6z+Qcv7W/AtwXs79+wxQ+yb+ScAzEfEG4LxcryMOJGYV6+ZtBCTtCPwl8O38WMC7Scmy4NW5f2s5ga8EDs712+ZAYla12pWbVlsxXwf+CahN4W4NPBsRq/Pj+vy+a3L/5v0rcv22OZCYVayNHknTJOKSanmkFtUXj3LIKLCvLb5qY1YhBaj4RGqrJOJvB46UdDiwEbAZqYeyhaSJuddRn9+3lvt3maSJwObA0x2chnskZpUbLri1EBFnRsSOETEVOAb4r4g4Dvg58P5cbWTu31pO4Pfn+h31SBxIzCqmiEJbCZ8GTpe0lDQHMjuXzwa2zuWnA2d0egAPbcyq1KM7pEXEdcB1+fcHgP1GqfMSaxOKl+JAYlYpf9fGzLqgH5bIO5CYVc09EjMrJUBDDiRmVtb4jyMOJGZVK3lpd0xwIDGrmgOJmZUSFFq1OtY5kJhVSJRetTomOJCYVc2BxMxKCcCXf82sLA9tzKw8BxIzK8df2jOzsgIHEjPrAq8jMbOyPNlqZuUEMDT+uyQOJGaV8mRr1z3HM0/+LK58qOp2tGEy8GTVjWjLf1fdgLaNv/cYXtdWbQeS7oqIbapuQzsk3doiz4iVNBDvcZcCiaSdgMuA15KmcGdFxPmStgLmAFNJHyUfiIhncnrO84HDgReAj0TEbZ0c2+kozKoUwHAU21pbDfxjRPwZsD9wqqQ9SGkmFuYk4gtZm3biMGBa3mYCF3V6Gg4kZpUKiOFiW6tXilhe61FExHPAPaT8vvXJwkcmEb8skptIGfm27+QsxtTQZhyaVXUDBkB/v8ftXbWZLOnWusezImLU90fSVODPgZuB7SJiOaRgI2nbXG1NEvGslmB8edEG1bhHUkKj/4k1koYk3S7pbkk/lDSp02NJOkjSNfn3IyU1zIomaQtJH+vgGJ+T9Mmi5SPqXCrp/c3qjKg/VdLdreq1eo/7QkSxLef+rdsaBZFNgB8BH4+IPzY5cteSiDuQ9NaLEbF3REwHVgIn1+9U0vb/g4i4OiK+0qTKFkDbgcQqUjyQtCRpfVIQ+UFE/DgXP1YbsuSfj+fyWhLxmvoE421xIFl3fgm8IX8S3yPpW8BtwE6SZki6UdJtueeyCYCkQyXdK+kG4K9rLyTpI5K+mX/fTtJVku7I2wHAV4Bdc2/oq7nepyT9WtKdkj5f91pnSbpP0s+A3VudhKS/za9zh6QfjehlvUfSLyUtkXRErj9B0lfrjv13Zd/I/lIwiBQIJPkqzGzgnoj4Wt2u+mThI5OIH58/0PYHVtSGQO3yHMk6IGkiaYb8J7lod+DEiPiYpMnA2cB7IuJ5SbWEz/8KXAy8G1hKunw3mguAX0TE+yRNADYhzcpPj4i98/FnkGbm9yN1Z6+WdCDwPClr/Z+T/i3cBixqcTo/joiL8+t+CTgJ+EbeNxV4J7Ar8HNJbwCOJ/0DfYukDYFfSfopfZGEoQsCGO7ayta3Ax8G7pJ0ey77DOmDZa6kk4CHWZvvdx7p0u9S0uXfEzs9sANJb21c9z/0l6RPix2Ah/IsOaTLdHuQ/sAANgBuBN4IPBgR9wNI+j7pEt1I7yb9sRIRQ8AKSVuOqDMjb7/JjzchBZZNgasi4oV8jKsLnNP0HEC2yK8zv27f3IgYBu6X9EA+hxnAnnXzJ5vnYy8pcKzB0KV1JBFxA6PPewAcPEr9AE7txrEdSHrrxVqvoCYHi+fri4AFEXHsiHp7071PbQFfjoj/GHGMj3dwjEuB90bEHZI+AhxUt2/ka0U+9mkRUR9walcVDPpiZavnSKp3E/D2PAxA0iRJuwH3ArtI2jXXO7bB8xcCp+TnTpC0GfAcqbdRMx/4aN3cy5R8CfB64H2SNpa0KfBXBdq7KbA8T+odN2Lf0ZLWy21+PXBfPvYpuT6SdpP0mgLHGQwRxNBQoW0sc4+kYhHxRP5kvzzPIQCcHRFLJM0ErpX0JHADMH2Ul/gHYFYe/w4Bp0TEjZJ+lS+v/r+I+JSkPwNuzD2iPwEfiojbJM0BbgceIg2/WvnfpLUJDwF38cqAdR/wC2A74OSIeEnSt0lzJ7flycAnWLsgyqDoqtUxTdEH3Sqz8WrzidvE2zY9qlDd+c/OXjRWv3fkHolZlSK6edWmMg4kZlXrg1GBA4lZxcI9EjMrx3dIM7OyAhjjl3aLcCAxq1AA0QeXfx1IzKoUUeimRWOdA4lZxfqhR+IFaWYVkvQT0p3yi3gyIg7tZXs65UBiZqX5S3tmVpoDiZmV5kBiZqU5kJhZaQ4kZlaaA4mZleZAYmalOZCYWWkOJGZW2v8HgoF8iHI0ciUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# display confusion matrix\n",
    "cm = confusion_matrix(target_test, target_predicted_dt)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Random Forest\n",
    "Using the same data, I built a random forest with 500 bootstrapped trees. Notice I parallelized this to 4 cores as big random forest can be computationally expensive. \n",
    "\n",
    "My overall results went up by 3% over the decision tree. Also, my minory target precision, but the recall decresed.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
      "            criterion='gini', max_depth=None, max_features='auto',\n",
      "            max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
      "            min_impurity_split=None, min_samples_leaf=1,\n",
      "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "            n_estimators=500, n_jobs=-1, oob_score=True, random_state=None,\n",
      "            verbose=0, warm_start=False)\n",
      "Random Forfest Accuracy 0.941\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      " Churn = no       0.94      0.99      0.97      1708\n",
      "Churn = yes       0.95      0.63      0.76       292\n",
      "\n",
      "avg / total       0.94      0.94      0.94      2000\n",
      "\n",
      "[[1698   10]\n",
      " [ 108  184]]\n"
     ]
    }
   ],
   "source": [
    "# train random forest model\n",
    "#paralleized to 4 cores \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(n_estimators= 500, n_jobs=-1,oob_score=True, class_weight=\"balanced\")\n",
    "print(rf)\n",
    "rf.fit(features_train, target_train)\n",
    "# test random forest model\n",
    "target_predicted_rf = rf.predict(features_test)\n",
    "print(\"Random Forfest Accuracy\", accuracy_score(target_test, target_predicted_rf))\n",
    "target_names = [\"Churn = no\", \"Churn = yes\"]\n",
    "print(classification_report(target_test, target_predicted_rf, target_names=target_names))\n",
    "print(confusion_matrix(target_test, target_predicted_rf))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Cross Validation of Random Forest\n",
    "I cross validated with 10 repeats. You can see the OOB score for each repeat and the mean. The mean is .949, which is quite close to the orginal model. I am not going to worry about over fitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross Validation Score for each K [0.94019934 0.93355482 0.94684385 0.96677741 0.93687708 0.94314381\n",
      " 0.94314381 0.93645485 0.94648829 0.9264214 ]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9419904665607396"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#verify RF with cross validation\n",
    "scores_rf = cross_val_score(rf, features_train, target_train, cv=10, n_jobs=-1)\n",
    "print(\"Cross Validation Score for each K\",scores_rf)\n",
    "scores_rf.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Visual of Confusion Matrix for Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1698   10]\n",
      " [ 108  184]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARIAAAD0CAYAAABXXakIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGvVJREFUeJzt3XuYXFWZ7/HvL4kkhDsEEEIwEQLK8AATICIeFcUJFxlxfHQGREXMMQcGOSrjKAhnwAujczwPKCODgiCiDpBBOYcDzMTIiIoPd+QWrgEOGAyEcAnILUn3e/5Yq0il01W9q3ZVdnfV78Ozn+7ae9Xeq5r022utvfZ6FRGYmZUxruoKmNnY50BiZqU5kJhZaQ4kZlaaA4mZleZAYmalOZCYWWkOJBWTtKGk/ytphaR/K3GeoyT9opN1q4qkd0p6oOp6WHHyhLRiJH0UOBF4C/AicAdwRkRcX/K8HwdOAPaPiNWlKzrKSQpgZkQsrrou1jkTqq7AWCDpROAk4FhgAbASOBg4HCgVSIA3AQ/2QxApQtKEfvpZHPSejeKZZwcKlb3trtcWRMTBXa5SeyLCW5MN2Az4E/CRJmUmAt8G/pi3bwMT87EDgCXA3wHLgKXAMfnYV0hBaVW+xlzgdOAndeeeDgQwIb/+JPAIqVX0KHBU3f7r6963P3ALsCJ/3b/u2HXA14Df5fP8ApjS4LPV6v/Fuvp/EDgUeBB4FvhyXfnZwA3A87nsd4EN8rHf5M/yUv68f1N3/i8BTwI/ru3L79kpX2NWfr09sBw4oOp/G53YZu0xMVYt3anQBtxadX0bbR4jGdnbgUnAFU3KnALsB+wF7En6ZTq17vgbSQFpKilYnCNpi4g4DfhH4LKI2DgiLmhWEUkbAWcDh0TEJqRgcccw5bYErs5ltwLOBK6WtFVdsY8CxwDbABsAX2hy6TeSfgZTgX8Azgc+BuwNvBP4B0lvzmUHgM8DU0g/uwOBvwWIiHflMnvmz3tZ3fm3JLXO5tVfOCIeJgWZn0qaDPwQuCgirmtS3zEkGIjBQtto5kAysq2A5dG8uX0U8NWIWBYRT5NaGh+vO74qH18VEdeQ/hrv2mZ9BoHdJW0YEUsjYtEwZd4PPBQRP46I1RFxCXA/8Jd1ZX4YEQ9GxCvAfFIQbGQVaTxoFXApKUh8JyJezNdfBOwBEBG3RcSN+br/D/g+8O4Cn+m0iHgt12ctEXE+8BBwE7AdKXD3hAAGiULbaOZAMrJngCmSmo0nbQ88Vvf6sbzv9XMMCUQvAxu3WpGIeInUHTgWWCrpaklvKVCfWp2m1r1+soX6PBMRtY587Rf9qbrjr9TeL2kXSVdJelLSC6QW15Qm5wZ4OiJeHaHM+cDuwD9HxGsjlB1TBgv+N5o5kIzsBuBV0rhAI38kNctrdsz72vESMLnu9RvrD0bEgoj4C9Jf5vtJv2Aj1adWpyfarFMrziXVa2ZEbAp8GdAI72n651bSxqRxpwuA03PXrScEwUAU20YzB5IRRMQK0rjAOZI+KGmypDdIOlXSs5IWk/7anyppa0lTcvmftHnJO4B3SdpR0mbAybUDkraV9IE8VvIaqYs03JD/NcAukj4qaYKkvwF2A65qs06t2AR4AfhTbi0dN+T4U8Cb13nXuraQtEzSPcB3gNsi4r+Sxn6+18kKV81dmz4REWeS5pCcCjwN/IH0Cz6P9As6hXQn5S7gbuB24OttXmshcFk+122s/cs/jnT354+kOxnvJg9kDjnHM8BhuewzpDsuh0XE8nbq1KIvkAZyXyS1li4bcvx04EeSnpf0103O8xLpFvsm+euxef+JwCxJR3Wy0lUJYIAotI1mnpDWBklvB06PiIPy65MBIuIblVasx0iaDlwVEbtXXJWu2XPPDWLBNSMNISXb7bD0tojYp8tVaotbJO2ZSmqV1Cxh7YFMs8IGC26jmQNJe4YbPHTTzloWBbs1Rbs2ki6sG1uq33+CpAckLZL0P+v2nyxpcT52UN3+g/O+xZJOGum6niLfniXAtLrXO9D+XRrrZwEDnf0TdBFpNvHFtR2S3kN6nGOPiHhN0jZ5/27AEcCfkaYM/FLSLvlt5wB/Qfq3foukKyPi3kYXdYukPbcAMyXNkLQB6X/GlRXXycagNCGtc12biPgNaSC+3nHAN2vzbyJiWd5/OHBpngj4KLCYNCt7NrA4Ih6JiJWkSYiHN7uuA0kb8uSyz5Ae4LsPmN9ghqm1SdIlpDk8u0paImlu1XXqDjFQcCNNjLy1bps30tmzXYB3SrpJ0q8l7Zv3Nxrra3kM0F2bNuWp7tdUXY9eFRFHVl2H9SGAweJdm+Vt3rWZAGxBeh5sX2B+fjaq0VjfcA2MprV0IDGrUAAru98xWAL8PNJcj5slDZLmPjUb62tpDNBdG7OKDYYKbSX8b+C9kJ6FIj3tvZw0rneEpImSZgAzgZtpYwzQLRKzCqWZraWCxFry2NIBpPGUJcBpwIXAhfmW8Erg6Nw6WSRpPnAvsBo4vvZwpqTaGOB44MKRxgA9s9WsQm/dY2JcfNV2hcrOftNjntnai1oYNbc29cPPeD10bbrOgaScnv9HPgr09M+41rUpePt31PIYiVmlxECM/b/noyqQTNlyfEyf9oaqq1HYjlMnsM+ek8bUINODd00eudAoMonJbKotx9TP+FVeYmW8VqgJEcAqxne5Rt03qgLJ9Glv4OYF00YuaG07aPtmS7NaJ9wU1xYuG+EWiZl1wOAoH/8owoHErEJpsNUtEjMrxV0bMyspLSPgQGJmJQRiZfiujZmVNOiujZmV4cFWMystEAOj/DmaIhxIzCrmwVYzKyUC3/41s7Lkma1mVk4AK2Ps/xqO/TaV2RgWFFvUqOjCRo0y7eVjX5AUkqbk15J0ds6md5ekWXVlj5b0UN6OHum6DiRmFRtgXKGtoIuAg4fulDSNlDnv8brdh5AWfJ5JWkDq3Fx2S9Jar28jJcs6TdIWzS7qQGJWoZTXZlyhrdD5hs+0B3AW8EXWzk9zOHBxJDcCm0vaDjgIWBgRz0bEc8BChglO9cZ+58xsTOv+MoqSPgA8ERF3Smtdy5n2zHpBrUVS0BRJt9a9Pi8izmv2BkmTgVOAOcMdblClRvsbciAxq1gLLZJ2UnbuBMwAaq2RHYDbJc2mcaa9JaTcOPX7r2t2EY+RmFUoQqwanFBoa+/8cXdEbBMR0yNiOilIzIqIJ0nZ8z6R797sB6yIiKWkxFhzJG2RB1nn5H0NuUViVqG0Hkl3M+1FxAUNil8DHAosBl4GjgGIiGclfY2UuhPgqxEx3ADu6xxIzCrV2RXSIuLIEY5Pr/s+gOMblLuQlOqzEAcSswqlwVZPkTezkrweiZmVUpsiP9Y5kJhVzOuRmFkpEbBq0IHEzEpIXRsHEjMrqdvP2qwPDiRmFfLtXzPrAHdtzKwDvGarmZWSVpF3IDGzEgKxetC5f82sJHdtzKwU37Uxs47wXRszK6eFnDWjmQOJWYU6vUJaVRxIzCrWCy2Ssd85MxvDAlg9OK7QVsRwKTslfUvS/Tkt5xWSNq87dnJO2fmApIPq9h+c9y2WdNJI1+1qIGm1Mmb9ptO5fxk+ZedCYPeI2AN4EDgZQNJuwBHAn+X3/Iuk8ZLGA+eQUnruBhyZyzbUtUDSTmXM+tEgKrQVMVzKzoj4RUSszi9vJOWpgZSy89KIeC0iHiWtJj87b4sj4pGIWAlcmss21M0WScuVMes7QSstkimSbq3b5rVxxU8B/56/HxMpO4erzNuGFso/jHkAO0712K/1lxYnpLWTae91kk4BVgM/re1qUKXhGhiVpewslD805y49D2CfPSc1raxZL1ofd20kHQ0cBhyY89lA45SdNNk/rG4GkmaVNDPSYOtAl9dslXQw8CXg3RHxct2hK4F/lXQmsD0wE7iZ1AiYKWkG8ARpQPajza7RzUByS6uVMetH3U7ZSbpLMxFYmBOJ3xgRx0bEIknzgXtJXZ7jI2Ign+czpHy/44ELI2JRs+t2LZBExOpWK2PWbyI627VpkLKzUe5fIuIM4Ixh9l9Dyg1cSFdHN1utjFk/ih6Y2erbJGaV8kN7ZtYBbpGYWSle2MjMyvPiz2ZWVuCujZmV5sFWM+uA6IEHQxxIzCrmro2ZlRLhQGJmHeAxEjMrbXDQgcTMSgjkro2ZldcDN20cSMwq5cFWM+uIHmiSNAwkkjZt9saIeKHz1THrP73QImm2WOQi4J78ddGQ1/c0eZ+ZtSDNJRl5K6JBpr0tJS2U9FD+ukXeL0ln5wR2d0maVfeeo3P5h/LC0U01DCQRMS0idsxfpw15vWOxj2VmzURADI4rtBV0Eetm2jsJuDYiZgLX5teQktfNzNs84FxIgYe01uvbSPmpTqsFn0YK1U7SEZK+nL/fQdLeRd5nZiPrZItkuEx7pMR0P8rf/wj4YN3+iyO5Edhc0nbAQcDCiHg2Ip4jpfwcGpzWMmIgkfRd4D3Ax/Oul4HvjfyRzKyQKLi1n2lv24hYCpC/bpP3r9dMe/tHxCxJv88VeVbSBsXqb2bNtTQhrVSmvWEvvq5osr+hIl2bVZLG1U4kaStgsMD7zKyI4i2Sdj2Vuyzkr8vy/kZJ7FpOblckkJwD/AzYWtJXgOuBfypSezMbQZ6QVmQr4UqgduflaOD/1O3/RL57sx+wInd9FgBzJG2RB1nn5H0Njdi1iYiLJd0GvC/v+khE+PavWad0cEJag0x73wTmS5oLPA58JBe/BjgUWEwa+zwGXh+++BopWybAVyNi6ADuWorObB0PrKJxpnIza1f3M+0BHDhM2QCOb3CeC4ELi163yF2bU4BLSEmGdyAlHT656AXMbATdHyPpuiItko8Be9eymEs6A7gN+EY3K2bWF4KOtkiqUiSQPDak3ATgke5Ux6z/9PTiz5LOIsXLl4FFkhbk13NId27MrBN6OZCw5sG8RcDVdftv7F51zPpQL3dtIuKC9VkRs74UoB6Y3jniGImknYAzgN2ASbX9EbFLF+tl1ifUEy2SInNCLgJ+SJp/fwgwH7i0i3Uy6y89cPu3SCCZHBELACLi4Yg4lfQ0sJl1Qg8EkiK3f1+TJOBhSccCT7DmMWQzK2uUB4kiigSSzwMbA/+dNFayGfCpblbKrG/0y4S0iLgpf/siaxY3MrMOUS+3SCRdQZNGV0R8qCs1Mus3vRxIgO+ut1pkD927CYfusc5DitZB43dummXEOkCPt7aAYE+3SCLi2vVZEbO+1Q9jJGbWRWPg1m4RDiRmVeuBQFJ4tTNJE7tZEbN+pSi2FTqX9HlJiyTdI+kSSZMkzZB0U86ad1ktC4Skifn14nx8erufocgKabMl3Q08lF/vKemf272gmQ3RoZmtkqaS5nvtExG7k5ZIPYK0WPtZOdPec8Dc/Ja5wHMRsTNwFiUWdS/SIjkbOAx4BiAi7sRT5M06Qvnp3yJbQROADSVNACYDS4H3Apfn40Mz7dUy8F0OHJhnsbesSCAZFxGPDdk30M7FzGwYoWLbSKeJeAL4X6SV4pcCK0jLoj4fEatzsfqsea9n1MvHVwBbtfMRigSSP0iaDYSk8ZI+BzzYzsXMbBgdStmZc9AcDswgLda+EemJ/eGuCG1k1GukyF2b40jdmx2Bp4Bf5n1m1gEtTEgbKWXn+4BHI+JpAEk/B/YnJQefkFsd9Vnzahn1luSu0Gasm4C8kCLP2iwjDdiYWTd07vbv48B+kiYDr5By2dwK/Ar4MGkdoaGZ9o4GbsjH/zPnumlZkRXSzmeYjxoRRTOhm1kjLdzaHfFUETdJuhy4HVgN/B44j7Tm8qWSvp731ZZRvQD4saTFpJZI2w2GIl2bX9Z9Pwn4K/IAjZl1QAcnpEXEaaQ0nfUeAWYPU/ZV1qTvLKVI1+ay+teSfgws7MTFzaw3Fn9uJ4/vDOBNna6ImY1dRcZInmNN42scqS91UjcrZdZXeuBZm6aBJM9y25O0TivAYLujumY2jA4OtlapadcmB40rImIgbz3wkc1GmR5YRb7IGMnNkmZ1vSZm/aoHAkmzNVtrM+H+C/BpSQ8DL5Gm1UZEOLiYlSR6o2vTbIzkZmAWa54UNLNO64Pcv4KUXW891cWsP/V4i2RrSSc2OhgRZ3ahPmb9p8cDyXhShr2xv8S12SjW62MkSyPiq+utJmb9qscDiVsiZt02Bm7tFtEskDjlndl60NN3bSKirZWSzKw1vT5GYmbrgwOJmZXSB2MkZtZlojfuarSzsJGZdVIHH9qTtLmkyyXdL+k+SW+XtKWkhTll58KctgIlZ+eUnXeVeTjXgcSsYp3M/Qt8B/iPiHgLaS2h+0gLkV2bU3Zey5qFyQ4BZuZtHnBuu5/BgcSsaoMFtxFI2hR4F3mV+IhYGRHPs3ZqzqEpOy+O5EZS/pvt2vkIDiRmVSrYGlGBTHvAm4GngR9K+r2kH0jaCNg2IpYC5K/b5PKvp+zM6tN5tsSDrWZV61ymvQmkpT9OyDluvkPz9ZU7lrLTLRKzinVwjGQJsCQibsqvLycFlqdqXZb8dVld+Wl1769P59kSBxKzqnXork1EPAn8QdKuedeBwL2sSc0J66bs/ES+e7MfsKLWBWqVuzZmFevwFPkTgJ9K2oCUYe8YUoNhvqS5pPzAtex61wCHAouBl3PZtjiQmFWpwzNbI+IOYLhxlHUews1ZIY7vxHUdSMwqJHr86V8zW0964Fmbrg22SrpQ0jJJ93TrGma9QBGFttGsm3dtLgIO7uL5zca+ondsRncc6V7XJiJ+I2l6t85v1iu8sFEH5Gm+8wAmjdu44tqYVaAHAknlE9Ii4ryI2Cci9tlg3KSqq2O23nX46d9KVN4iMetrfZCy08zWh1He2iiim7d/LwFuAHaVtCRPzzWzOsJdm6Yi4shundusp4zyOSJFuGtjVrHR3toowoHErEpjYLJZEQ4kZhXzXRszK82BxMzKCTzYambl9cJga+VT5M36Xoef/pU0PqejuCq/niHpppxp77K8DCOSJubXi/Px6e1+BAcSswp1aULaZ0kZ9mr+CTgrZ9p7DqhNDp0LPBcROwNn5XJtcSAxq1JE8a0ASTsA7wd+kF8LeC8pNQWsm2mvloHvcuDAXL5lDiRmFdNgsa2gbwNfZE2Sz62A5yNidX5dn03v9Ux7+fiKXL5lDiRmFetUyk5JhwHLIuK2+t3DXDIKHGuJ79qYVSmAwcK/uyOl7HwH8AFJhwKTgE1JLZTNJU3IrY76bHq1THtLJE0ANgOebf1DuEViVr3OZdo7OSJ2iIjpwBHAf0bEUcCvgA/nYkMz7dUy8H04l2+rReJAYlax9bCMwJeAEyUtJo2BXJD3XwBslfefSPOE4025a2NWtS7MbI2I64Dr8vePALOHKfMqa9J3luJAYlaxXpjZ6kBiViEFqPhg66jlQGJWNT/9a2ZljfZ0nEU4kJhVySukmVl5xZ+jGc0cSMwq5rs2ZlaeWyRmVkqABhxIzKyssR9HHEjMqubbv2ZWngOJmZUSeGarmZUjwl0bM+sABxIzKyUA3/41s7LctTGz8nogkHjNVrNKdS5BlqRpkn4l6T5JiyR9Nu/fUtLCnLJzoaQt8n5JOjun7LxL0qx2P4UDiVmVgk5m2lsN/F1EvBXYDzhe0m6kRZ2vzSk7r2XNIs+HADPzNg84t92P4UBiVrXBgtsIImJpRNyev3+RlP93Kmun5hyasvPiSG4k5b/Zrp2P4DESs4q1MNg6RdKtda/Pi4jzhj2nNB34c+AmYNuIWAop2EjaJhd7PWVnVkvnubRw5TMHErMqBTBQeGrrSJn2AJC0MfAz4HMR8UKTvOAdS9npro1ZpTo32Aog6Q2kIPLTiPh53v1UrcuSvy7L+2spO2vq03m2ZFS1SF5YvXz5gqe//1jV9WjBFGB51ZVoydNVV6BlY+9nDG9qqXSHbv8qNT0uAO6LiDPrDtVSc36TdVN2fkbSpcDbgBW1LlCrRlUgiYitq65DKyTdWqSpae3ri59x5+aRvAP4OHC3pDvyvi+TAsh8SXOBx1mTXe8a4FBgMfAycEy7Fx5VgcSs7wTQoQRZEXE9w497ABw4TPkAju/EtR1IzCoVEGN/HQEHknKGvfVmHdXbP+PW7tqMWr5rU0Kje/g1kgYk3SHpHkn/Jmlyu9eSdICkq/L3H5B0UpOym0v62zaucbqkLxTdP6TMRZI+3MK1pku6Z6RyI/2Me0IH79pUxYGku16JiL0iYndgJXBs/cH8rEPL/w8i4sqI+GaTIpsDLQcSq4gDibXgt8DO+S/xfZL+BbgdmCZpjqQbJN2eWy4bA0g6WNL9kq4HPlQ7kaRPSvpu/n5bSVdIujNv+5NG6XfKraFv5XJ/L+mW/HDWV+rOdYqkByT9Eth1pA8h6dP5PHdK+tmQVtb7JP1W0oOSDsvlx0v6Vt21/1vZH2Rv6ew8kqo4kKwHkiaQHpC6O+/alfSMw58DLwGnAu+LiFnArcCJkiYB5wN/CbwTeGOD058N/Doi9gRmAYtID2U9nFtDfy9pDunBrNnAXsDekt4laW/gCNJU6g8B+xb4OD+PiH3z9e4D5tYdmw68G3g/8L38GeaS5ifsm8//aUkzClynPwQwOFhsG8U82NpdG9bdz/8tabLQ9sBj+SEpSE9p7gb8Lk9l3gC4AXgL8GhEPAQg6SekJzSHei/wCYCIGABW1B4TrzMnb7/PrzcmBZZNgCsi4uV8jSsLfKbdJX2d1H3aGFhQd2x+RAwCD0l6JH+GOcAedeMnm+VrP1jgWv1hlLc2inAg6a5XImKv+h05WLxUvwtYGBFHDim3F51LnSTgGxHx/SHX+Fwb17gI+GBE3Cnpk8ABdceGnivytU+IiPqAU3uozKAnAom7NtW7EXiHpJ0BJE2WtAtwPzBD0k653JEN3n8tcFx+73hJmwIvklobNQuAT9WNvUzNT4D+BvgrSRtK2oTUjRrJJsDS/EzHUUOOfUTSuFznNwMP5Gsfl8sjaRdJGxW4Tn+IIAYGCm2jmVskFYuIp/Nf9kskTcy7T42IByXNA66WtBy4Hth9mFN8FjgvT38eAI6LiBsk/S7fXv33PE7yVuCG3CL6E/CxiLhd0mXAHcBjpO7XSP4H6dH0x0hjPvUB6wHg18C2wLER8aqkH5DGTm7Pz4I8zZr1MAw6NrO1SooeaFaZjVWbTdg63r7J4YXKLnj+gttG63NHbpGYVSli1N+RKcKBxKxqPdArcCAxq1i4RWJm5Yz+WatFOJCYVSmAUX5rtwgHErMKBRA9cPvXgcSsSuGFjcysA3qhReIJaWYVkvQfpJXyi1geEQd3sz7tciAxs9L80J6ZleZAYmalOZCYWWkOJGZWmgOJmZXmQGJmpTmQmFlpDiRmVpoDiZmV9v8BLmDaLXwF478AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# display confusion matrix\n",
    "cm = confusion_matrix(target_test, target_predicted_rf)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning\n",
    "You can tune any argument in these models. I did a grid search only on max_features (mtry in R). I parallelized the job to 4 cores for speed. You can see that max_features (mtry) of 5 had the best results. But frankly was very little difference from the other parameter results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to run 2.9032600000000004 seconds\n",
      "{'mean_fit_time': array([2.13039668, 2.37619241, 2.33483442, 1.66196648]), 'std_fit_time': array([0.00582594, 0.15498902, 0.48455955, 0.07433556]), 'mean_score_time': array([0.24279364, 0.29023449, 0.20359874, 0.20310195]), 'std_score_time': array([0.04731158, 0.04031142, 0.00029264, 0.00115545]), 'param_max_features': masked_array(data=[2, 3, 4, 5],\n",
      "             mask=[False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'max_features': 2}, {'max_features': 3}, {'max_features': 4}, {'max_features': 5}], 'split0_test_score': array([0.87712288, 0.89110889, 0.9000999 , 0.90709291]), 'split1_test_score': array([0.886, 0.901, 0.909, 0.919]), 'split2_test_score': array([0.88588589, 0.8958959 , 0.90690691, 0.91391391]), 'mean_test_score': array([0.883     , 0.896     , 0.90533333, 0.91333333]), 'std_test_score': array([0.00415913, 0.00403971, 0.00380063, 0.00487954]), 'rank_test_score': array([4, 3, 2, 1], dtype=int32), 'split0_train_score': array([1., 1., 1., 1.]), 'split1_train_score': array([1., 1., 1., 1.]), 'split2_train_score': array([1., 1., 1., 1.]), 'mean_train_score': array([1., 1., 1., 1.]), 'std_train_score': array([0., 0., 0., 0.])}\n"
     ]
    }
   ],
   "source": [
    "# use a full grid over all parameters\n",
    "param_grid = {\"max_features\": [2, 3, 4, 5]}\n",
    "start_time = time.clock()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# run grid search\n",
    "grid_search = GridSearchCV(rf, param_grid=param_grid,n_jobs=-1)\n",
    "\n",
    "grid_search.fit(features_train, target_train)\n",
    "\n",
    "print(\"Time to run\", time.clock() - start_time, \"seconds\")\n",
    "print(grid_search.cv_results_)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#KNN\n",
    "I performed KNN on K=3 and K=5. For both K's the accurancy was 85% and 87% respectively and I still have problems with the minority class. KNN and Decision Tree perform about the same. I find this to be true frequently, which is why I use them as my base comparative models. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More Details\n",
    "Now that we know our random forest was the best model of the three I ran, I will gather some other information. Below is a non-ordered list of feature importance. I only showed 20 for purposes of space."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Show importance of each feature in Random Forest\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features sorted by their score:\n",
      "Higher the more important\n",
      "[(0.1239, 'number_customer_service_calls'), (0.1216, 'total_day_minutes'), (0.1205, 'total_day_charge'), (0.05, 'total_eve_minutes'), (0.0492, 'total_eve_charge'), (0.0388, 'total_night_minutes'), (0.0385, 'total_night_charge'), (0.0363, 'total_intl_charge'), (0.0355, 'total_intl_minutes'), (0.0355, 'account_length'), (0.0334, 'total_intl_calls'), (0.0311, 'total_night_calls'), (0.0303, 'total_eve_calls'), (0.0303, 'total_day_calls'), (0.0184, 'number_vmail_messages'), (0.002, 'state_AZ'), (0.001, 'state_AR'), (0.0007, 'state_AL'), (0.0006, 'state_AK')]\n"
     ]
    }
   ],
   "source": [
    "print(\"Features sorted by their score:\")\n",
    "print(\"Higher the more important\")\n",
    "print(sorted(zip(map(lambda x: round(x, 4), rf.feature_importances_),df.columns[1:20]), \n",
    "             reverse=True))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC curve for Random Forest\n",
    "Finally a ROC curve that shows the lift I get from the Random Forest model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC: 0.921\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XlYVdX6wPHvCwiI4ARqJhrOoDibOZTZoJnXyhxSs7Kyn6mVaeZtMBu0uW6ZV8vKypvdrmXm0KhZpmWaYc6zqSmpqYiATDK8vz/OkVARDsThwOH9PM95Onvvtfd+2eF+WWvtvZaoKsYYY8z5+Hg6AGOMMaWbJQpjjDH5skRhjDEmX5YojDHG5MsShTHGmHxZojDGGJMvSxTGGGPyZYnCeBUR2SciqSJyUkQOi8gsEQk+q0xnEflORJJEJEFEPhORZmeVqSwiU0Rkv/NYu53LYec5r4jIaBHZLCLJIhIrInNFpIU7f15jSoIlCuONrlPVYKA10AZ45PQGEekELAEWAhcC9YENwEoRaeAs4w98CzQHegKVgc5AHNDhPOd8DbgfGA1UB5oAC4B/FDZ4EfEr7D7GuJPYm9nGm4jIPuAuVV3qXH4RaK6q/3Au/wBsUtVRZ+33FXBUVW8TkbuAZ4CGqnrShXM2BrYDnVR1zXnKfA98oKozncu3O+O81LmswL3AGMAPWAycVNUHcx1jIbBcVV8RkQuBfwNdgZPAq6o61YVLZEyhWY3CeC0RCQeuBXY7l4Nw1Azm5lH8Y6C78/vVwNeuJAmnq4DY8yWJQugDXAI0Az4EBoqIAIhINaAHMEdEfIDPcNSE6jjPP0ZErvmb5zcmT5YojDdaICJJwAHgCPCEc311HL/zh/LY5xBwuv8h9Dxlzqew5c/nOVU9rqqpwA+AApc5t/UHVqnqQeBioIaqTlLVU6q6B3gbGFQMMRhzDksUxhv1UdUQoBsQyV8JIB7IBmrnsU9t4Jjze9x5ypxPYcufz4HTX9TRJjwHGOxcdTPwX+f3i4ALReTE6Q/wKFCrGGIw5hyWKIzXUtXlwCzgZedyMrAKGJBH8ZtwdGADLAWuEZFKLp7qWyBcRNrnUyYZCMq1fEFeIZ+1/D+gv4hchKNJap5z/QFgr6pWzfUJUdVeLsZrTKFYojDebgrQXURaO5cfBoY6H2UNEZFqIvI00Al4yllmNo6b8TwRiRQRHxEJFZFHReScm7Gq7gJeB/4nIt1ExF9EAkVkkIg87Cy2HugrIkEi0ggYVlDgqroOOArMBBar6gnnpjVAoog8JCIVRcRXRKJF5OKiXCBjCmKJwng1VT0KvA9MdC7/CFwD9MXRr/A7jkdoL3Xe8FHVdBwd2tuBb4BEHDfnMODn85xqNDANmA6cAH4DbsTR6QzwKnAK+BP4D381IxXkf85YPsz1M2UB1+F4/HcvjiazmUAVF49pTKHY47HGGGPyZTUKY4wx+bJEYYwxJl+WKIwxxuTLEoUxxph8lbnBx8LCwjQiIsLTYRhjTJmydu3aY6paoyj7lrlEERERQUxMjKfDMMaYMkVEfi/qvtb0ZIwxJl+WKIwxxuTLEoUxxph8WaIwxhiTL0sUxhhj8mWJwhhjTL7clihE5F0ROSIim8+zXURkqojsFpGNItLWXbEYY4wpOnfWKGYBPfPZfi3Q2PkZDrzhxliMMabcSs/M+lv7u+2FO1VdISIR+RS5AXjfOeXjahGpKiK1VbU45h42xphyKztb+XnvcY4np/POBx+x5MvPCt4pH558M7sOueYIBmKd685JFCIyHEetg3r16pVIcMYYU9akZWSxYN0fvP3DHnbs3sPxb2aQ+tsvhNVrTPLfOK4nE4XksS7PWZRU9S3gLYD27dvbTEvGGK+Rla3EnUznz8R0/kxM48+kNP5MTOeo879/JqZx7GQ6WdkFHyv1VCbJp7JoVjuEgO+nIIf2MHHyczz+8INUqFChyDF6MlHEAnVzLYcDBz0UizHGuE3cyXTWHzjBloOJHEpIOycJZJ/1568IhFYKoGZIALUqB9CsdmUq+BXcpXxo5wZu7ncpV7e8iPVd/0NYWBh169YtcL+CeDJRLALuFZE5wCVAgvVPGGPKulOZ2Ww9lMj6/fGsO3CC9QdO8HtcSs720Er+1KwcmJMAalYOcCyHBOSsDwsOoIKv688axcXF8fDDDzNr5kwuSn2C7q2epE2bNsX2M7ktUYjI/4BuQJiIxAJPABUAVHUG8CXQC9gNpAB3uCsWY4w5LT75FFsOJnIkKa3YjpmZpWw/nMS6A/FsOZjIqUxHO1GtygG0qVuNwR3q0aZuVVqEVyHIv/huu6rK+++/z4MPPkh8fDzjx49n/PjxxXb809z51NPgArYrcI+7zm+MKd9Uldj4VLYcTGTroUS2HkzIafpxhwA/H1qGV+H2zhG0rluVNvWqUrtKRbec67SHHnqIl156ic6dOzNjxgxatGjhlvOUufkojDHmtMysbBLTMklIzeBEyin2Hkt2JAZnckhIzQDAR6BhjWA61K9O8wsr0/zCKtSpWhHJ65GaIhCE2lUDC9VcVFSpqakkJycTFhbGsGHDaNy4McOGDcPHx33ntkRhjPGo7GwlKT2TxNQM5w3f8d+E1AxOpJ4iITWDxLPXpzjWJaVnnnO8AD8fImtXpleL2s6kUJnICypT0d/XAz9d8fr666+55557aN26NfPmzaNp06Y0bdrU7ee1RGGMcasTKaf4PS6FfXHJ/B6X4vwkc/Rkek4SOPupn9z8fX2oElSBKhUdn1qVA2laK4TKFf9aV9W5vW71IBqEVcKvBP6yL0kHDx5kzJgxzJ07l6ZNm3LvvfeW6PktURhTzqgqe44ls27/iZxO1+KSrcqfiWnscyaD3+NScpp/TrugciAXhQbRKrxqzg3+nE9QBapW9KdKxQoEVvBBiquNqAz69ttvufHGGzl16hSTJ09m/PjxBAQElGgMliiM8XKqyu9xKazaE8eq3+JYvSeOI0npbjufr49Qp2pFLgoN4rpWtYkIrUS96kFEhDn+G1ih7DcBlYSMjAwqVKhAq1at6NWrF08//TSNGjXySCyWKIwppVSV5FNZxCefymmbz1bXByY4dCItJzkcTnQ86VMjJIBODULp1DCUiyOqERJY9Ld1z6d6Jf8S6dT1VomJiUycOJGff/6ZlStXEhYWxpw5czwakyUKY0pA6qksTqSeIj7Z0UF7IsXRIRuf4kgC8cmnOOF8cic+5XTH7Skysv7eiDVhwf50bBBKR2dyaBBWqVw345Rmqsonn3zC/fffz+HDhxk1ahTp6ekEBQV5OjRLFMb8HUeS0jhwPIUjiekcSUrnSFJazvejSenEJadzIiWD9Hz6AgIr+FAtyNEeXy3In8Y1g6ka5E/VoApUc7bVVw2qQOWKFfDzcf0mXzXIn4Y1LDGUBUePHmXo0KF89dVXtGnThoULF3LxxRd7OqwcliiMKYS0jCzW7D3O8p1HWbHzKLuOnDxju6+PEBbsT82QQGpXCSS6TmWqBfmfceOvUtGfapX+SgDWZm8qV67MsWPHmDJlCvfccw9+fqXr1ly6ojHGzbKztVDt/ArsO5bsSAy7jvHznjjSM7Px9/PhkvrVGdA+nMa1QqgZEkDNkECqV/LHtxB/9Zvya8WKFTzzzDPMmzeP4OBgVq9e7daX5v4OSxTGq6SeymLt7/EciHc0B/2ZlMaRxPxH6nRVwxqVuPmSenRtUoOO9UO94gUuU/KOHTvG+PHjmTVrFhEREezbt4/o6OhSmyTAEoXxAgeOp7BsxxG+236EVb/FndEfcPZInTVCAghwYbjm3MJCAriscRjh1TzfqWjKLlXlvffeY/z48SQmJvLII4/w2GOPlYrO6oJYojAlJiElg8OJafk2/ahCYloGsfGpxMan5Pz34Ik0MvKYuSUjSzl20vFOQERoEDdfUo/Lm9SgSa0QwoID8C9kUjDGnT744AOaNWvGjBkzaN68uafDcZklClMoR5PS2fxHAruOJJFZQBtOYmomB46n8PvxZPbHpZCYdu64PPkRgVohgYRXq0jrulUJrJD3TT/ygspcEVmT+mGVCnV8Y9wtJSWFZ599lhEjRhAeHs68efOoUqVKqW5myoslinIgO1v5YfcxPo45wJHEog2xrAqx8ak5L265ooKvULdaEHWrB9G2XjXqVQ/igiqBBT7iWSnAj7rVgqhdNZAAP+sHMGXTl19+yT333MO+ffuoU6cOI0eOpFq1ap4Oq0gsUZRxaRlZLN95NM8xezKyslmz9zjfbj/C0aR0qlfyJ/KCkCKf65IG1WlRpwot6lQhsnblAtv6K/j62BNAptyJjY1lzJgxzJs3j6ioKJYvX07Xrl09HdbfYomiDFu24whPLtpyxjSLZwsJ8KNr0xpcG30B3ZvVsr/QjXGzZ555hi+++IJnn32WcePG4e/v7+mQ/jbRQjxTXhq0b99eY2JiPB3G36KqbD2UWOiROxNSM1iw7g9+3H2MzGzlREoGDWpUYkKvKC4KzevJCaFe9SDr0DXGzdasWUPFihVp0aIFcXFxJCQk0KBBA0+HdQYRWauq7Yuyr9UoStjWg4k8sWgzv+yLL9L+IYF+dG9Wi+AAP+qHVWLIJRdZIjDGQxISEnj00Ud544036N27N4sWLSI0NJTQ0FBPh1asLFG4QXJ6JqkZWTnL3+84ygerfyczO5utBxOpGuTPpBuaU7d64Z6f9vMR2l9U3V70MsbDVJWPPvqIsWPHcuTIEe677z4mT57s6bDcxhJFAbY4J2R3VezxFGYs38Ops575j7wghAurVqRjl1DuvbIRVYPKfrulMeXVBx98wG233Ub79u35/PPPadeunadDcqtynyiS0zN58evtHD157kQuqvDttiPn3PQLcnVULS5vEpazXDXIn14tatsTQMaUYenp6ezZs4eoqChuuukmMjMzue222/D19f4afrlNFKcys3nh6+288+NefAQa1gjOs1ynhqFM7B3l8gifFXx9qBkSYEM7G+NFli1bxsiRI0lJSWHXrl0EBARwxx13eDqsElNuE8X7q/bxzo97AZg5tD1XRtbybEDGmFLnyJEjPPjgg8yePZsGDRrw1ltvlfh81aVBuUkU6/bH8/jCLaQ5O5lPNzX9/OhV1Koc6MnQjDGl0O7du+nQoQMnT55kwoQJTJgwgYoVK3o6LI/w2kSRnpnFwnUHSc90JIbZq3/neHIGHeo7XqFvXCuYTg3DLEkYY86QmJhI5cqVadiwIcOGDePOO+8kKirK02F5lNcmioXrD/LPeRtzlv18hGk3t6FndG0PRmWMKa2Sk5OZNGkSb7/9Nhs3biQ8PJyXXnrJ02GVCl6VKGLjU1iy5U8mfb4VgJohAXwx+jJEwN/Ph8qBFTwcoTGmNPrss8+499572b9/P8OGDSsTc0SUJK9KFCM/+JVNfyQAcE3zWtzaMYIaIeWv48kY45rMzExuuukm5s+fT/Pmzfnhhx+49NJLPR1WqeM1ieJQQipbDibQoX51nr0xmkY1iz5KqjHGu6kqIoKfnx+1a9fm+eefZ+zYsV4xgJ87eM0gQQdPpJKtcM8VjSxJGGPOa/Xq1bRv355ff/0VgOnTp/PQQw9ZksiH1ySK0+w1N2NMXuLj4xk5ciSdO3fmzz//JD6+aANzlkduTRQi0lNEdojIbhF5OI/t9URkmYisE5GNItKrqOf6bMMhABsmwxhzjo8++ojIyEjeeustxowZw7Zt27jqqqs8HVaZ4bY+ChHxBaYD3YFY4BcRWaSqW3MVewz4WFXfEJFmwJdARGHPlZaRxayf9gHQ7qKyOdWgMcZ9tm/fTkREBF9//TVt2rTxdDhljjtrFB2A3aq6R1VPAXOAG84qo0Bl5/cqwMGinGj9gRMAdG1Sw+UxmYwx3istLY2nnnqKzz77DIBHH32Un376yZJEEbkzUdQBDuRajnWuy+1J4BYRicVRm7gvrwOJyHARiRGRmKNHj56xLStbeeTTTQT5+/LKTa2KLXhjTNm0dOlSWrZsyZNPPsny5csBqFChQrkY5dVd3Jko8uosOHve1cHALFUNB3oBs0XknJhU9S1Vba+q7WvUqHHGth93H2PvsWS6Nq5BWLC9M2FMefXnn38yZMgQunfvjqqyZMkSXn75ZU+H5RXcmShigbq5lsM5t2lpGPAxgKquAgKBMArhz4Q0AB7rXb7HYjGmvPvmm2/45JNPePzxx9m0aRPdu3f3dEhew50v3P0CNBaR+sAfwCDg5rPK7AeuAmaJSBSORHGUIrD5H4wpfzZs2MCuXbvo378/Q4YMoUuXLtSvX9/TYXkdt9UoVDUTuBdYDGzD8XTTFhGZJCLXO4uNA/5PRDYA/wNuV9Wzm6eMMeYMJ0+eZNy4cbRr146HH36YzMxMRMSShJu4dQgPVf0SRyd17nWP5/q+FejizhiMMd5lwYIF3HfffcTGxjJ8+HCee+45/Py8ZjSiUqnMX93EtAxPh2CMKSGbNm3ixhtvpEWLFnz00Ud07tzZ0yGVC2V6CI8jSWk8/cU2ggP8qGmjxBrjlTIyMvjuu+8AaNGiBV988QVr1661JFGCynSi2HowEYD7rmxEBd8y/aMYY/Lw008/0a5dO7p3787u3bsB6NWrFxUq2NwyJalM311//d0xqFeLOlU8HIkxpjgdP36c4cOH06VLF06cOMGnn35Ko0aNPB1WuVWm+yje+XEvABfXr+7hSIwxxSUtLY3WrVtz8OBBxo0bx5NPPklwcLCnwyrXymyi2Bh7guRTWdzZpb41OxnjBWJjYwkPDycwMJDJkyfTunVrWrWyYXlKgzJ7hz095eldl9lz08aUZampqTz++OM0bNgwZxC/oUOHWpIoRVyqUYiIP1BPVXe7OR6XHU1KB7A5sY0pw5YsWcKoUaP47bffuOWWW+jQoYOnQzJ5KLBGISL/ADYB3ziXW4vIfHcHVpDUU1kE+PlYs5MxZdR9993HNddcg4+PD0uXLmX27NnUqlXL02GZPLhSo5gEXAIsA1DV9SJSKh4/sOGdjClbsrKyAPD19aVjx46EhYXx0EMPERgY6OHITH5c+XM8Q1VPnLXO4+Mx/XEilWpBNhm6MWXFr7/+SqdOnXj99dcBGDJkCE888YQliTLAlUSxTURuAnxEpL6ITAFWuzmuAv30WxwXVq3o6TCMMQVISkpi7NixXHzxxezfv5/atWt7OiRTSK4kinuBdkA28CmQBtzvzqBckZyeyWHnXBTGmNJpyZIlREVF8dprr3H33Xezfft2+vfv7+mwTCG50kdxjao+BDx0eoWI9MWRNDzG38+Hq6NqejIEY0wB/P39qVmzJvPmzeOSSy7xdDimiFypUTyWx7oJxR1IYfmI2GRFxpQyGRkZvPDCC0yY4LhFdOvWjZiYGEsSZdx5axQicg3QE6gjIq/k2lQZRzOUMcbk+PHHHxkxYgRbtmxhwIABZGdn4+Pjg4+PPcJe1uX3f/AIsBlHn8SWXJ8lwLXuD80YUxbExcVx1113cdlll5GUlMRnn33Gxx9/bAnCi5y3RqGq64B1IvJfVbVeY2NMnuLi4pgzZw7//Oc/efzxx6lUqZKnQzLFzJXO7Doi8gzQDMh54FlVm7gtKmNMqbZt2zY+/vhjnnjiCZo0acL+/fupXt1GcfZWrtQNZwHvAYKjyeljYI4bYzLGlFIpKSlMmDCBVq1a8dprrxEbGwtgScLLuZIoglR1MYCq/qaqjwFXuDes/GVnKwmpNle2MSXp66+/Jjo6mmeffZabb76ZHTt2EB4e7umwTAlwpekpXRzPof4mIiOAPwCPvsCwfNdRAGpXsVf/jSkJJ0+e5NZbbyU0NJRly5bRrVs3T4dkSpArNYqxQDAwGugC/B9wpzuDKsi+Y8kA9GphQwEY4y5ZWVl88MEHZGVlERwczNKlS9mwYYMliXKowBqFqv7s/JoE3AogIh6tbx47mY6vj1C3epAnwzDGa61du5a7776btWvXUrFiRfr162cTCZVj+dYoRORiEekjImHO5eYi8j4eHhQwI0vx87G3so0pbgkJCYwePZoOHTrwxx9/MGfOHPr27evpsIyHnTdRiMhzwH+BIcDXIjIBx5wUGwCPPhp7LCmdsGCb2c6Y4tavXz+mTZvGqFGj2L59OwMHDrShcky+TU83AK1UNVVEqgMHncs7Sia084tPOUX1SjYXhTHFYc+ePdSoUYOQkBCeeeYZfHx8uPjiiz0dlilF8mt6SlPVVABVPQ5sLw1JAiBbwceanoz5W06dOsWzzz5L8+bNefrppwG45JJLLEmYc+RXo2ggIqeHEhcgItcyquqxhssdh5OoVdmanowpqhUrVjBixAi2bdtG//79GT16tKdDMqVYfomi31nL09wZSGGcTM8kO8Hjs7EaUya9+uqrPPDAA0RERPDFF1/Qq1cvT4dkSrn8BgX8tiQDKQx/Px+uiqrl6TCMKTOys7NJTk4mJCSEf/zjHxw9epTHHnuMoCB7xNwUrEyOA+wjjo8xpmBbtmzh8ssv5/bbbwegSZMmPPvss5YkjMvcmihEpKeI7BCR3SLy8HnK3CQiW0Vki4h86M54jClPUlJSeOSRR2jdujXbtm2jd+/eqFqTrSk8V8Z6AkBEAlQ1vRDlfYHpQHcgFvhFRBap6tZcZRoDjwBdVDVeRGwSbGOKwbp16+jbty/79u3jjjvu4MUXXyQsLMzTYZkyqsAahYh0EJFNwC7ncisR+bcLx+4A7FbVPap6CsfQ5DecVeb/gOmqGg+gqkcKFb0x5gynawz16tWjXr16LF++nHfffdeShPlbXGl6mgr0BuIAVHUDrg0zXgc4kGs51rkutyZAExFZKSKrRaSnC8clK1uxl0WN+UtmZiZTpkzhqquuIisri9DQUJYvX07Xrl09HZrxAq4kCh9V/f2sdVku7JfXrfzsBlI/oDHQDRgMzBSRquccSGS4iMSISMyRo8eIT8mgVogNMW4MwJo1a+jQoQNjx44lMDCQxMRET4dkvIwrieKAiHQAVER8RWQMsNOF/WKBurmWw3EMA3J2mYWqmqGqe4EdOBLHGVT1LVVtr6rtK1d1zKR1UZjNy2vKt5MnT3LPPffQsWNH/vzzT+bOncsXX3xBtWrVPB2a8TKuJIqRwANAPeBPoKNzXUF+ARqLSH0R8QcGAYvOKrMAZzOWc4TaJsCe/A6acioTgPBqFV0IwRjvVaFCBb7//nvuu+++nDesbQA/4w6uPPWUqaqDCntgVc0UkXuBxYAv8K6qbhGRSUCMqi5ybushIltxNGeNV9W4/I57KisbPyxRmPJp9+7dTJo0ienTpxMSEsLatWsJDLRmWONertQofhGRL0VkqIiEFObgqvqlqjZR1Yaq+oxz3ePOJIE6PKCqzVS1harOKeiYp/9eqml9FKYcSU9PZ/LkyURHR7NgwQLWr18PYEnClIgCE4WqNgSeBtoBm0RkgYgUuoZhjCmaZcuW0apVKx5//HH69OnD9u3bueyyyzwdlilHXHozW1V/UtXRQFsgEceERsYYN1NVnnnmGTIyMvj666+ZM2cOF154oafDMuVMgX0UIhKM40W5QUAUsBDo7Oa4jCm3srOzeeedd+jZsyd169Zl9uzZVK1alYoVrV/OeIYrNYrNOJ50elFVG6nqOFX92c1xGVMubdy4kUsvvZThw4czc+ZMAGrXrm1JwniUK089NVDVbLdH4iJVGznWeJ+TJ0/y1FNP8eqrr1KtWjVmzZrFbbfd5umwjAHySRQi8i9VHQfME5Fzhpz01Ax3ccmnuLRGsCdObYzbPPnkk/zrX//irrvu4vnnnyc0NNTTIRmTI78axUfO/5aame3AUZtIzXBlBBFjSrcDBw6QnJxMZGQkDz/8MH369OHSSy/1dFjGnOO8fRSqusb5NUpVv839wdGp7RGC0KWhjYRpyq7MzExeeeUVoqKiuPvuuwEICwuzJGFKLVc6s+/MY92w4g7EmPJg9erVtG/fnnHjxtGtWzf+85//eDokYwqUXx/FQByPxNYXkU9zbQoBTrg7MGO8zRdffMF1113HhRdeyKeffkqfPn1sbCZTJuTXR7EGxxwU4ThmqjstCVjnzqCM8RaqysGDB6lTpw5XX301kyZN4v777yckpFCj4RjjUedNFM5hv/cCS0suHGO8x86dOxk1ahQ7d+5k69atBAcH89hjj3k6LGMK7bx9FCKy3PnfeBE5nusTLyLHSy5EY8qWtLQ0nnzySVq0aEFMTAyPPPKIvTBnyrT8mp5OT3dqjxgZ46LDhw/TtWtXdu3axeDBg3nllVe44IILPB2WMX9Lfo/Hnn4buy7gq6pZQCfgbsCmlzMml4yMDABq1apF165dWbJkCR9++KElCeMVXHk8dgGOaVAbAu/jeIfiQ7dGZUwZkZ2dzYwZM2jYsCGxsbGICDNnzqR79+6eDs2YYuNKoshW1QygLzBFVe8D6rg3LGNKvw0bNtC5c2dGjhxJ48aNc2oVxngbVxJFpogMAG4FPneuq+C+kIwp3VSVBx98kHbt2rFnzx5mz57N0qVLqV+/vqdDM8YtXH0z+wocw4zvEZH6wP/cG5YxpZeIEB8fz7Bhw9ixYwe33HKLvThnvJorU6FuBkYDMSISCRw4Pf+1MeXF77//Tp8+ffj1118BePvtt3nzzTepVq2ahyMzxv0KTBQichmwG3gHeBfYKSJd3B3Y+SjnjHhujNtkZGTw4osv0qxZM7755ht27NgBgI+PS7MIG+MVXJm46FWgl6puBRCRKGA20N6dgZ1PtkKlAFfCNubv+emnn7j77rvZvHkzN9xwA1OnTqVevXqeDsuYEufKHdf/dJIAUNVtIuLvxpjyla1KWIjHTm/KkaVLl5KQkMCCBQu44YYbPB2OMR4jqvk35YjILCAdRy0CYAgQpKpD3Rta3gJqN9a3P/2G2zpFeOL0xoupKrNnz6ZGjRpce+21pKenk5GRQXCwzahoyj4RWauqRWoJcqWhdQTwG/BP4CFgD463sz3Gz9qHTTHbvn07V155JUOHDuW9994DICAgwJKEMRTQ9CQiLYCGwHxVfbFkQirYhVUDPR2C8RKpqak8++yzvPDCC1SqVIk333yTu+5dNpsKAAAenklEQVS6y9NhGVOq5Dd67KM4hu8YAnwjInnNdOcRlSva+36meHz22Wc8/fTTDBw4kO3btzN8+HB7osmYs+RXoxgCtFTVZBGpAXyJ4/FYY8q0w4cPs379enr27MmAAQOIiIigQ4cOng7LmFIrvz+d0lU1GUBVjxZQ1phSLysri9dff52mTZty6623kpqaiohYkjCmAPnVKBrkmitbgIa5585W1b5ujcyYYvTrr78yYsQIfvnlF66++mpef/11m0zIGBfllyj6nbU8zZ2BGOMue/fupUOHDoSFhfHhhx8yaNAgG5vJmELIb87sb0syEGOKk6qyadMmWrZsSf369Xnvvfe47rrrqFq1qqdDM6bMsX4H43X27t1L7969adOmDRs3bgTg1ltvtSRhTBG5NVGISE8R2SEiu0Xk4XzK9RcRFRGPjB9lvMOpU6d4/vnnad68OcuXL+fll1+mWbNmng7LmDLP5dH1RCRAVdMLUd4XmA50B2KBX0RkUe5xo5zlQnAMY/6zq8c25mxZWVl07tyZtWvX0rdvX6ZMmULdunU9HZYxXsGVYcY7iMgmYJdzuZWI/NuFY3cAdqvqHlU9BcwB8hpZbTLwIpDmetjGOCQmJgLg6+vLnXfeyWeffca8efMsSRhTjFxpepoK9AbiAFR1A44Z7wpSBziQazmWs+baFpE2QF1V/Zx8iMhwEYkRkRgXzmvKAVVl1qxZNGjQgIULFwIwatQoevfu7eHIjPE+riQKH1X9/ax1WS7sl9fzhzlD1YqID465LsYVdCBVfUtV2xd15EPjXbZu3Uq3bt244447iIyMpGHDhp4OyRiv5kqiOCAiHQAVEV8RGQPsdGG/WCB3/T8cOJhrOQSIBr4XkX1AR2CRdWib/Lz44ou0atWKzZs3M3PmTFasWEF0dLSnwzLGq7mSKEYCDwD1gD9x3NBHurDfL0BjEanvnOhoELDo9EZVTVDVMFWNUNUIYDVwvapa85I5x+l5Uy644AKGDBnC9u3bGTZsmA3gZ0wJKPBfmaoeUdVBzpt6mPP7MRf2ywTuBRYD24CPVXWLiEwSkev/fuimPDh48CADBgzg3/92PD9x2223MWvWLGrUqOHhyIwpPwp8PFZE3iZX38Jpqjq8oH1V9Usco87mXvf4ecp2K+h4pvw4PYDfhAkTyMjIoHPnzp4OyZhyy5X3KJbm+h4I3MiZTzMZU6zWr1/PXXfdxdq1a+nRowevv/66dVgb40EFJgpV/Sj3sojMBr5xW0Sm3EtISODgwYN89NFHDBgwwAbwM8bDXH4zO5f6wEXFHYgpv1SVuXPnsmvXLiZMmMDll1/Onj17CAy0KW+NKQ1ceTM7XkSOOz8ncNQmHnV/aKY8+O233+jVqxcDBw5k4cKFZGRkAFiSMKYUyTdRiKPO3wqo4fxUU9UGqvpxSQRnvFd6ejrPPPMM0dHRrFy5ktdee42ffvqJChVsPnRjSpt8m55UVUVkvqq2K6mATPlw4MABJk+ezHXXXceUKVOoU6dOwTsZYzzClbeV1ohIW7dHYrze0aNHmTbNMVFio0aN2Lp1K3PnzrUkYUwpd95EISKnaxuX4kgWO0TkVxFZJyK/lkx4xhtkZ2fzzjvvEBkZyQMPPMCOHTsAaNCggYcjM8a4Ir+mpzVAW6BPCcXisiB/X0+HYFy0efNmRo4cyY8//shll13GjBkzaNq0qafDMsYUQn6JQgBU9bcSisVlNYIDPB2CccGpU6fo0aMHp06d4t133+X222+3dyKMKYPySxQ1ROSB821U1VfcEE+BfESoFuTviVMbF3333Xdcfvnl+Pv78/HHHxMZGUlYWJinwzLGFFF+ndm+QDCO4cDz+niEv68PPj72V2lpFBsbS79+/bjqqqt4//33Abj00kstSRhTxuVXozikqpNKLBJTZmVmZjJt2jQmTpxIVlYWzz33HEOGDPF0WMaYYlJgH4UxBbn11luZM2cO1157LdOnT6d+/fqeDskYU4zk9IQw52wQqa6qx0s4ngJVqRupCQe2ezqMcu/EiRP4+fkRHBzMjz/+yOHDh+nXr591VhtTSonI2qJOJ33ePorSmCSM56kqc+bMISoqiokTJwKOfoj+/ftbkjDGS9k8ksZlu3fv5pprrmHw4MGEh4dzyy23eDokY0wJsERhXPLhhx8SHR3Nzz//zLRp01i9ejXt2tkQYMaUB0WZj8KUIxkZGVSoUIH27dvTv39/XnzxRS688EJPh2WMKUHn7cwurawzu2QcOXKEcePGkZyczKeffurpcIwxf5NbOrNN+ZSdnc1bb71F06ZN+eijj2jevDlZWVmeDssY40HW9GRy7Nmzh1tuuYVVq1bRrVs33njjDSIjIz0dljHGwyxRmBxVqlThxIkT/Oc//+HWW2+1x12NMYA1PZV7ixYtom/fvmRlZREaGsrmzZu57bbbLEkYY3JYoiin9u/fT58+fbjhhhvYuXMnhw4dAsDHx34ljDFnsrtCOZOZmcnLL79MVFQUS5Ys4YUXXmDdunWEh4d7OjRjTCllfRTlTFZWFjNnzuTKK6/k3//+NxEREZ4OyRhTylmNohyIj4/noYceIikpiYCAAFauXMmiRYssSRhjXGKJwoupKv/973+JjIzkX//6F8uWLQMgNDTUOquNMS6zROGldu7cSffu3bnllluIiIggJiaG66+/3tNhGWPKIOuj8FJjxowhJiaG119/neHDh+Pr6+vpkIwxZZQlCi/yzTffEBkZSd26dXnjjTcICAjgggsu8HRYxpgyzq1NTyLSU0R2iMhuEXk4j+0PiMhWEdkoIt+KyEXujMdbHT58mJtvvpkePXrwwgsvAHDRRRdZkjDGFAu3JQoR8QWmA9cCzYDBItLsrGLrgPaq2hL4BHjRXfF4o+zsbGbMmEFkZCTz5s3jiSee4OWXX/Z0WMYYL+POGkUHYLeq7lHVU8Ac4IbcBVR1maqmOBdXA/bWVyE899xzjBw5knbt2rFx40aefPJJAgMDPR2WMcbLuLOPog5wINdyLHBJPuWHAV/ltUFEhgPDAYJqNyyu+MqkpKQkjh07Rv369RkxYgT169dn8ODB9rirMcZt3FmjyOvOlecsSSJyC9AeeCmv7ar6lqq2V9X2fr7ls/9dVZk/fz7NmjVj4MCBqCqhoaHcfPPNliSMMW7lzkQRC9TNtRwOHDy7kIhcDUwArlfVdDfGU2b9/vvvXH/99fTt25fq1aszdepUSw7GmBLjzj/PfwEai0h94A9gEHBz7gIi0gZ4E+ipqkfcGEuZtWrVKq6++moAXn75Ze6//378/MpnrcoY4xluq1GoaiZwL7AY2AZ8rKpbRGSSiJx+RfglIBiYKyLrRWSRu+IpaxITEwFo27Ytd955J9u2bWPcuHGWJIwxJU5U8+w2KLWq1I3UhAPbPR2G28TFxfHwww+zZMkStmzZQnBwsKdDMsZ4ARFZq6rti7KvjfVUSqgq77//PpGRkbz33nsMHDjQ+iGMMaWCtWOUAgkJCfTp04fvv/+eTp06MWPGDFq2bOnpsIwxBrBE4VGqiohQuXJlwsLCeOuttxg2bJhNR2qMKVXsjuQhixcvpm3btsTGxiIizJ07l//7v/+zJGGMKXXsrlTCDh06xKBBg+jZsycpKSkcOWJPBRtjSjdLFCVo+vTpREZGsmDBAp566ik2btxI27ZtPR2WMcbky/ooStDatWu55JJLmD59Oo0bN/Z0OMYY4xKrUbhRYmIiY8aMYe3atQC8/vrrLF682JKEMaZMsUThBqrKJ598QlRUFFOnTmX58uUABAYG2rsRxpgyxxJFMdu7dy+9e/dmwIAB1KxZk1WrVvHAAw94OixjjCkySxTF7L///S8rVqzg1Vdf5ZdffuGSS/KbgsMYY0o/G+upGPzwww+kp6dz9dVXk56eztGjRwkPt8n6jDGlh4315CHHjh3jzjvvpGvXrkyaNAmAgIAASxLGGK9ij8cWgaoya9Ysxo8fT0JCAg899BATJ070dFjGQzIyMoiNjSUtLc3ToRhDYGAg4eHhVKhQodiOaYmiCL788kvuvPNOunTpwowZM4iOjvZ0SMaDYmNjCQkJISIiwp5qMx6lqsTFxREbG0v9+vWL7bjW9OSilJQUVq5cCUCvXr1YuHAhK1assCRhSEtLIzQ01JKE8TgRITQ0tNhrt5YoXPDVV18RHR3Ntddey4kTJxARrr/+ehvAz+SwJGFKC3f8LtqdLh9//PEHAwYMoFevXgQEBPDZZ59RtWpVT4dljDElyhLFeRw5coRmzZrx+eef8/TTT7NhwwYuv/xyT4dlTJ58fX1p3bo10dHRXHfddZw4cSJn25YtW7jyyitp0qQJjRs3ZvLkyeR+LP6rr76iffv2REVFERkZyYMPPuiJHyFf69at46677vJ0GPl67rnnaNSoEU2bNmXx4sV5lvnuu+9o27Yt0dHRDB06lMzMTMDx/lXLli1p2bIlnTt3ZsOGDTn73HnnndSsWfOcZu4HH3yQ7777zn0/UG6qWqY+lcObqjvFxsbmfH/ttdd09+7dbj2fKfu2bt3q6RC0UqVKOd9vu+02ffrpp1VVNSUlRRs0aKCLFy9WVdXk5GTt2bOnTps2TVVVN23apA0aNNBt27apqmpGRoZOnz69WGPLyMj428fo37+/rl+/vkTPWRhbtmzRli1balpamu7Zs0cbNGigmZmZZ5TJysrS8PBw3bFjh6qqTpw4UWfOnKmqqitXrtTjx4+rquqXX36pHTp0yNlv+fLlunbtWm3evPkZx9u3b5927949z3jy+p0EYrSI91176skpISGBxx57jDfffJPVq1fTtm1bRo8e7emwTBnz1Gdb2HowsViP2ezCyjxxXXOXy3fq1ImNGzcC8OGHH9KlSxd69OgBQFBQENOmTaNbt27cc889vPjii0yYMIHIyEgA/Pz8GDVq1DnHPHnyJPfddx8xMTGICE888QT9+vUjODiYkydPAvDJJ5/w+eefM2vWLG6//XaqV6/OunXraN26NfPnz2f9+vU5TbeNGjVi5cqV+Pj4MGLECPbv3w/AlClT6NKlyxnnTkpKYuPGjbRq1QqANWvWMGbMGFJTU6lYsSLvvfceTZs2ZdasWXzxxRekpaWRnJzMd999x0svvcTHH39Meno6N954I0899RQAffr04cCBA6SlpXH//fczfPhwl69vXhYuXMigQYMICAigfv36NGrUiDVr1tCpU6ecMnFxcQQEBNCkSRMAunfvznPPPcewYcPo3LlzTrmOHTsSGxubs9y1a1f27dt3zjkvuugi4uLiOHz4MBdccMHfir8g5T5RqCpz585lzJgxHD58mHvvvZeGDRt6OixjiiQrK4tvv/2WYcOGAY5mp3bt2p1RpmHDhpw8eZLExEQ2b97MuHHjCjzu5MmTqVKlCps2bQIgPj6+wH127tzJ0qVL8fX1JTs7m/nz53PHHXfw888/ExERQa1atbj55psZO3Ysl156Kfv37+eaa65h27ZtZxwnJibmjGaXyMhIVqxYgZ+fH0uXLuXRRx9l3rx5AKxatYqNGzdSvXp1lixZwq5du1izZg2qyvXXX8+KFSvo2rUr7777LtWrVyc1NZWLL76Yfv36ERoaesZ5x44dy7Jly875uQYNGsTDDz98xro//viDjh075iyHh4fzxx9/nFEmLCyMjIwMYmJiaN++PZ988gkHDhw45/jvvPMO1157bYHXF6Bt27asXLmSfv36uVS+qMp1olBV+vbty4IFC2jbti2LFi2iffsiveFuDECh/vIvTqmpqbRu3Zp9+/bRrl07unfvDvw1L3teCvN0zNKlS5kzZ07OcrVq1QrcZ8CAAfj6+gIwcOBAJk2axB133MGcOXMYOHBgznG3bt2as09iYiJJSUmEhITkrDt06BA1atTIWU5ISGDo0KHs2rULESEjIyNnW/fu3alevToAS5YsYcmSJbRp0wZw1Ip27dpF165dmTp1KvPnzwfgwIED7Nq165xE8eqrr7p2ceCMPp/Tzr6+IsKcOXMYO3Ys6enp9OjRAz+/M2/By5Yt45133uHHH3906bw1a9bk4MGDLsdZVOUyUWRkZFChQgVEhEsvvZQrr7ySUaNG5fxSG1PWVKxYkfXr15OQkEDv3r2ZPn06o0ePpnnz5qxYseKMsnv27CE4OJiQkBCaN2/O2rVrc5p1zud8CSf3urOf3a9UqVLO906dOrF7926OHj3KggULeOyxxwDIzs5m1apVVKxYMd+fLfexJ06cyBVXXMH8+fPZt28f3bp1y/OcqsojjzzC3Xfffcbxvv/+e5YuXcqqVasICgqiW7dueb53UJgaRXh4+Bm1g9jYWC688MJz9u3UqRM//PAD4EhkO3fuzNm2ceNG7rrrLr766qtzktb5pKWl5Xvtiku5e+rp+++/p2XLlixcuBCAcePGcd9991mSMF6hSpUqTJ06lZdffpmMjAyGDBnCjz/+yNKlSwFHzWP06NH885//BGD8+PE8++yzOTes7OxsXnnllXOO26NHD6ZNm5azfLrpqVatWmzbti2nael8RIQbb7yRBx54gKioqJwb4dnHXb9+/Tn7RkVFsXv37pzlhIQE6tSpA8CsWbPOe85rrrmGd999N6cP5Y8//uDIkSMkJCRQrVo1goKC2L59O6tXr85z/1dffZX169ef8zk7SQBcf/31zJkzh/T0dPbu3cuuXbvo0KHDOeWOHDkCQHp6Oi+88AIjRowAYP/+/fTt25fZs2fn9GG4YufOnSXy0m+5SRRHjx5l6NChXHHFFaSnp59RtTXGm7Rp04ZWrVoxZ84cKlasyMKFC3n66adp2rQpLVq04OKLL+bee+8FoGXLlkyZMoXBgwcTFRVFdHQ0hw4dOueYjz32GPHx8URHR9OqVaucv7Sff/55evfuzZVXXknt2rXzjWvgwIF88MEHOc1OAFOnTiUmJoaWLVvSrFkzZsyYcc5+kZGRJCQkkJSUBMA///lPHnnkEbp06UJWVtZ5z9ejRw9uvvlmOnXqRIsWLejfvz9JSUn07NmTzMxMWrZsycSJE8/oWyiq5s2bc9NNN9GsWTN69uzJ9OnTc/747NWrV07z0EsvvURUVBQtW7bkuuuu48orrwRg0qRJxMXFMWrUKFq3bn1GE/jgwYPp1KkTO3bsIDw8nHfeeQdwtIzs3r27RJrLy8Uw4//73/+45557OHnyJOPHj2fChAkEBQW5KUJT3mzbto2oqChPh+HVXn31VUJCQkr9uxQlaf78+fz6669Mnjz5nG15/U7aMOMFyMzMJDo6mvXr1/PMM89YkjCmjBk5ciQBAQGeDqNUyczMdOmJteLglTWK5ORkJk+eTL169Rg1alTOEwk2Ho9xB6tRmNLGahQF+Pzzz2nevDkvvPBCTgediFiSMG5V1v7gMt7LHb+LXpMoYmNj6du3L9dddx2VKlVixYoVTJkyxdNhmXIgMDCQuLg4SxbG49Q5H0VgYGCxHtdr3qPYs2cPixcv5rnnnuOBBx7A39/f0yGZciI8PJzY2FiOHj3q6VCMyZnhrjiV6T6KNWvWsGrVKu6//37AMZaKqy+qGGNMeVJq+yhEpKeI7BCR3SJyzlsqIhIgIh85t/8sIhGuHPfEiROMGjWKjh078sorr5CcnAxgScIYY9zAbYlCRHyB6cC1QDNgsIg0O6vYMCBeVRsBrwIvFHTcjJREIiMjefPNNxk9ejSbNm0647V9Y4wxxcudNYoOwG5V3aOqp4A5wA1nlbkB+I/z+yfAVVLA40mpxw9Tt25dfvnlF6ZMmULlypWLPXBjjDF/cWdndh0g9xi6scAl5yujqpkikgCEAsdyFxKR4cDpAePTY2JiNp89dHI5FcZZ16ocs2vxF7sWf7Fr8ZemRd3RnYkir5rB2T3nrpRBVd8C3gIQkZiidsh4G7sWf7Fr8Re7Fn+xa/EXEYkp6r7ubHqKBermWg4Hzh44PaeMiPgBVYDjbozJGGNMIbkzUfwCNBaR+iLiDwwCFp1VZhEw1Pm9P/CdlrXndY0xxsu5renJ2edwL7AY8AXeVdUtIjIJxyTfi4B3gNkishtHTWKQC4d+y10xl0F2Lf5i1+Ivdi3+YtfiL0W+FmXuhTtjjDEly2vGejLGGOMeliiMMcbkq9QmCncN/1EWuXAtHhCRrSKyUUS+FZGLPBFnSSjoWuQq119EVES89tFIV66FiNzk/N3YIiIflnSMJcWFfyP1RGSZiKxz/jvp5Yk43U1E3hWRIyKy+TzbRUSmOq/TRhFp69KBVbXUfXB0fv8GNAD8gQ1As7PKjAJmOL8PAj7ydNwevBZXAEHO7yPL87VwlgsBVgCrgfaejtuDvxeNgXVANedyTU/H7cFr8RYw0vm9GbDP03G76Vp0BdoCm8+zvRfwFY532DoCP7ty3NJao3DL8B9lVIHXQlWXqWqKc3E1jndWvJErvxcAk4EXgbSSDK6EuXIt/g+YrqrxAKp6pIRjLCmuXAsFTo/3U4Vz3+nyCqq6gvzfRbsBeF8dVgNVRaR2QcctrYkir+E/6pyvjKpmAqeH//A2rlyL3Ibh+IvBGxV4LUSkDVBXVT8vycA8wJXfiyZAExFZKSKrRaRniUVXsly5Fk8Ct4hILPAlcF/JhFbqFPZ+ApTeiYuKbfgPL+DyzykitwDtgcvdGpHn5HstRMQHxyjEt5dUQB7kyu+FH47mp244apk/iEi0qp5wc2wlzZVrMRiYpar/EpFOON7filbVbPeHV6oU6b5ZWmsUNvzHX1y5FojI1cAE4HpVTS+h2EpaQdciBIgGvheRfTjaYBd5aYe2q/9GFqpqhqruBXbgSBzexpVrMQz4GEBVVwGBOAYMLG9cup+crbQmChv+4y8FXgtnc8ubOJKEt7ZDQwHXQlUTVDVMVSNUNQJHf831qlrkwdBKMVf+jSzA8aADIhKGoylqT4lGWTJcuRb7gasARCQKR6Ioj3PXLgJucz791BFIUNVDBe1UKpue1H3Df5Q5Ll6Ll4BgYK6zP3+/ql7vsaDdxMVrUS64eC0WAz1EZCuQBYxX1TjPRe0eLl6LccDbIjIWR1PL7d74h6WI/A9HU2OYsz/mCaACgKrOwNE/0wvYDaQAd7h0XC+8VsYYY4pRaW16MsYYU0pYojDGGJMvSxTGGGPyZYnCGGNMvixRGGOMyZclClPqiEiWiKzP9YnIp2zE+UbKLOQ5v3eOPrrBOeRF0yIcY4SI3Ob8fruIXJhr20wRaVbMcf4iIq1d2GeMiAT93XOb8ssShSmNUlW1da7PvhI67xBVbYVjsMmXCruzqs5Q1fedi7cDF+badpeqbi2WKP+K83Vci3MMYInCFJklClMmOGsOP4jIr85P5zzKNBeRNc5ayEYRaexcf0uu9W+KiG8Bp1sBNHLue5VzDoNNzrH+A5zrn5e/5gB52bnuSRF5UET64xhz67/Oc1Z01gTai8hIEXkxV8y3i8i/ixjnKnIN6CYib4hIjDjmnnjKuW40joS1TESWOdf1EJFVzus4V0SCCziPKecsUZjSqGKuZqf5znVHgO6q2hYYCEzNY78RwGuq2hrHjTrWOVzDQKCLc30WMKSA818HbBKRQGAWMFBVW+AYyWCkiFQHbgSaq2pL4OncO6vqJ0AMjr/8W6tqaq7NnwB9cy0PBD4qYpw9cQzTcdoEVW0PtAQuF5GWqjoVx1g+V6jqFc6hPB4DrnZeyxjggQLOY8q5UjmEhyn3Up03y9wqANOcbfJZOMYtOtsqYIKIhAOfquouEbkKaAf84hzepCKOpJOX/4pIKrAPxzDUTYG9qrrTuf0/wD3ANBxzXcwUkS8Al4c0V9WjIrLHOc7OLuc5VjqPW5g4K+EYriL3DGU3ichwHP+ua+OYoGfjWft2dK5f6TyPP47rZsx5WaIwZcVY4E+gFY6a8DmTEqnqhyLyM/APYLGI3IVjWOX/qOojLpxjSO4BBEUkz/lNnGMLdcAxyNwg4F7gykL8LB8BNwHbgfmqquK4a7scJ45Z3J4HpgN9RaQ+8CBwsarGi8gsHAPfnU2Ab1R1cCHiNeWcNT2ZsqIKcMg5f8CtOP6aPoOINAD2OJtbFuFogvkW6C8iNZ1lqovrc4pvByJEpJFz+VZgubNNv4qqfomjozivJ4+ScAx7npdPgT445kj4yLmuUHGqagaOJqSOzmarykAykCAitYBrzxPLaqDL6Z9JRIJEJK/amTE5LFGYsuJ1YKiIrMbR7JScR5mBwGYRWQ9E4pjycSuOG+oSEdkIfIOjWaZAqpqGY3TNuSKyCcgGZuC46X7uPN5yHLWds80CZpzuzD7ruPHAVuAiVV3jXFfoOJ19H/8CHlTVDTjmx94CvIujOeu0t4CvRGSZqh7F8UTW/5znWY3jWhlzXjZ6rDHGmHxZjcIYY0y+LFEYY4zJlyUKY4wx+bJEYYwxJl+WKIwxxuTLEoUxxph8WaIwxhiTr/8HF9BHuJhpUY4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Determine the false positive and true positive rates\n",
    "fpr, tpr, _ = roc_curve(target_test, rf.predict_proba(features_test)[:,1]) \n",
    "    \n",
    "# Calculate the AUC\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('ROC AUC: %0.3f' % roc_auc)\n",
    " \n",
    "# Plot of a ROC curve for a specific class\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Random Forest does the best, but I still am not getting the accurancy on my target class of interest. I have a few tricks I can do to work on this, but that is for another day/class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Other Ensembles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BaggingClassifier(base_estimator=None, bootstrap=True,\n",
      "         bootstrap_features=False, max_features=1.0, max_samples=1.0,\n",
      "         n_estimators=100, n_jobs=1, oob_score=False, random_state=0,\n",
      "         verbose=0, warm_start=False)\n",
      "Bagging Accuracy 0.9515\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         No       0.96      0.99      0.97      1708\n",
      "        Yes       0.91      0.74      0.82       292\n",
      "\n",
      "avg / total       0.95      0.95      0.95      2000\n",
      "\n",
      "[[1686   22]\n",
      " [  75  217]]\n"
     ]
    }
   ],
   "source": [
    "#Bagging Classifer\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "clf_bag = BaggingClassifier(n_estimators=100, random_state=0)\n",
    "print(clf_bag)\n",
    "clf_bag.fit(features_train, target_train)\n",
    "predicted_bag=clf_bag.predict(features_test)\n",
    "expected = target_test\n",
    "print(\"Bagging Accuracy\", accuracy_score(expected,predicted_bag))\n",
    "print(classification_report(expected, predicted_bag,target_names=['No', 'Yes']))\n",
    "print(confusion_matrix(expected, predicted_bag))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extra Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extra Trees 0.8295\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         No       0.92      0.88      0.90      1708\n",
      "        Yes       0.43      0.54      0.48       292\n",
      "\n",
      "avg / total       0.85      0.83      0.84      2000\n",
      "\n",
      "[[1502  206]\n",
      " [ 135  157]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "xdt = ExtraTreesClassifier(max_depth=3,\n",
    "                         n_estimators=10,class_weight='balanced')\n",
    "xdt.fit(features_train, target_train)\n",
    "predicted_xdt=xdt.predict(features_test)\n",
    "expected = target_test\n",
    "print(\"Extra Trees\", accuracy_score(expected,predicted_xdt))\n",
    "print(classification_report(expected, predicted_xdt,target_names=['No', 'Yes']))\n",
    "print(confusion_matrix(expected, predicted_xdt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaBoost of a Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaboost Accuracy 0.9475\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         No       0.96      0.98      0.97      1708\n",
      "        Yes       0.88      0.74      0.80       292\n",
      "\n",
      "avg / total       0.95      0.95      0.95      2000\n",
      "\n",
      "[[1680   28]\n",
      " [  77  215]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=3),\n",
    "                         algorithm=\"SAMME\",\n",
    "                         n_estimators=200)\n",
    "bdt.fit(features_train, target_train)\n",
    "predicted_bdt=bdt.predict(features_test)\n",
    "expected = target_test\n",
    "print(\"Adaboost Accuracy\", accuracy_score(expected,predicted_bdt))\n",
    "print(classification_report(expected, predicted_bdt,target_names=['No', 'Yes']))\n",
    "print(confusion_matrix(expected, predicted_bdt))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boost Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boost Accuracy 0.862\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         No       0.92      0.92      0.92      1708\n",
      "        Yes       0.53      0.53      0.53       292\n",
      "\n",
      "avg / total       0.86      0.86      0.86      2000\n",
      "\n",
      "[[1569  139]\n",
      " [ 137  155]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "clf_GBC = GradientBoostingClassifier(n_estimators=100, learning_rate=1.7, max_depth=1, random_state=0)\n",
    "clf_GBC.fit(features_train, target_train)\n",
    "predicted_GBC=clf_GBC.predict(features_test)\n",
    "expected = target_test\n",
    "print(\"Gradient Boost Accuracy\", accuracy_score(expected,predicted_GBC))\n",
    "print(classification_report(expected, predicted_GBC,target_names=['No', 'Yes']))\n",
    "print(confusion_matrix(expected, predicted_GBC))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "#Three Models RF, NB, BDT\n",
    "clf1 = RandomForestClassifier(random_state=1)\n",
    "clf2 = LogisticRegression()\n",
    "bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=3),\n",
    "                         algorithm=\"SAMME\",\n",
    "                         n_estimators=200)\n",
    "eclf2 = VotingClassifier(estimators=[('rf', clf1), ('gnb', clf2), ('bdt', bdt)], voting='hard')\n",
    "for MV, label in zip([clf1, clf2, bdt, eclf2], ['Random Forest', 'Logistic', 'AdaBoost Decision Tree', 'Ensemble']):\n",
    "\n",
    "    scores2 = cross_val_score(MV, features_train, target_train, cv=5, scoring='accuracy')\n",
    "    print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores2.mean(), scores2.std(), label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}