WNS-adClick_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/lightgbm/__init__.py:46: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_8.3.3) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import model_selection, preprocessing, metrics\n",
    "from tqdm import tqdm_notebook\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "import lightgbm\n",
    "import seaborn as sns\n",
    "import xgboost\n",
    "\n",
    "from sklearn import neural_network\n",
    "from scipy.stats import hmean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be04319fa1384fcf98df1671ce9a703e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tqdm_notebook().pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/Users/aakashkerawat/autoML/wns_adclick/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t = pd.read_csv(\"/Users/aakashkerawat/autoML/wns_adclick/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_v = pd.read_csv(\"/Users/aakashkerawat/autoML/wns_adclick/view_log.csv\")\n",
    "df_i = pd.read_csv(\"/Users/aakashkerawat/autoML/wns_adclick/item_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"impression_time\"] = pd.to_datetime(df[\"impression_time\"], format=\"%Y-%m-%d %H:%M:%S\")\n",
    "df_t[\"impression_time\"] = pd.to_datetime(df_t[\"impression_time\"], format=\"%Y-%m-%d %H:%M:%S\")\n",
    "df_v[\"server_time\"] = pd.to_datetime(df_v[\"server_time\"], format=\"%Y-%m-%d %H:%M:%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_v = df_v.sort_values([\"user_id\", \"server_time\"]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Full df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_v[\"dummy_ones\"] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_v[\"visit_number\"] = df_v.groupby(\"user_id\")[\"dummy_ones\"].cumsum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b68d30ac81449b38bf4403bc8b0258d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/tqdm/_tqdm.py:660: FutureWarning: Currently, 'apply' passes the values as ndarrays to the applied function. In the future, this will change to passing it as Series objects. You need to specify 'raw=True' to keep the current behaviour, and you can pass 'raw=False' to silence this warning\n",
      "  result = getattr(df, df_function)(wrapper, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:3: FutureWarning: set_axis currently defaults to operating inplace.\n",
      "This will change in a future version of pandas, use inplace=True to avoid this warning.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "session_number = df_v.groupby(\"user_id\")[\"session_id\"].expanding().progress_apply(lambda x: len(np.unique(x)))\n",
    "\n",
    "session_number.set_axis(df_v.index)\n",
    "\n",
    "df_v[\"session_number\"] = session_number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb14d05e7f75424cb88d11feccdaf166",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:3: FutureWarning: set_axis currently defaults to operating inplace.\n",
      "This will change in a future version of pandas, use inplace=True to avoid this warning.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "item_number = df_v.groupby(\"user_id\")[\"item_id\"].expanding().progress_apply(lambda x: len(np.unique(x)))\n",
    "\n",
    "item_number.set_axis(df_v.index)\n",
    "\n",
    "df_v[\"item_number\"] = item_number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "views_cols = [\"user_id\", \"server_time\",\"visit_number\", \"session_number\", \"item_number\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(\"impression_time\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t = df_t.sort_values(\"impression_time\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train test split fe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_v_tr = df_v.loc[df_v[\"server_time\"]<datetime(2018, 12, 3)].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tr = df.loc[df[\"impression_time\"]<datetime(2018, 12, 3)].reset_index(drop=True)\n",
    "df_te = df.loc[df[\"impression_time\"]>=datetime(2018, 12, 3)].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_v_tr = df_v_tr.sort_values([\"server_time\"]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_v = df_v.sort_values([\"server_time\"]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_f_tr = pd.merge_asof(left=df_tr, right=df_v_tr[views_cols], left_on=\"impression_time\", right_on=\"server_time\", direction=\"backward\", by=[\"user_id\"])\n",
    "\n",
    "df_f_te = pd.merge_asof(left=df_te, right=df_v_tr[views_cols], left_on=\"impression_time\", right_on=\"server_time\", direction=\"backward\", by=[\"user_id\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f = pd.merge_asof(left=df, right=df_v[views_cols], left_on=\"impression_time\", right_on=\"server_time\", direction=\"backward\", by=[\"user_id\"])\n",
    "\n",
    "df_ft = pd.merge_asof(left=df_t, right=df_v[views_cols], left_on=\"impression_time\", right_on=\"server_time\", direction=\"backward\", by=[\"user_id\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_combined_features(df_train, df_test):\n",
    "    train_ids = df_train[\"impression_id\"].unique()\n",
    "    test_ids = df_test[\"impression_id\"].unique()\n",
    "    df_combined = pd.concat([df_train ,df_test]).reset_index(drop=True)\n",
    "    df_combined[\"impression_time_hour\"] =  df_combined[\"impression_time\"].dt.hour\n",
    "    df_combined[\"impression_time_dom\"] =  df_combined[\"impression_time\"].dt.day\n",
    "\n",
    "    df_combined = df_combined.sort_values([\"user_id\", \"impression_time\"]).reset_index(drop=True)\n",
    "\n",
    "    df_combined[\"user_click_ewm\"] = df_combined.groupby(\"user_id\")[\"is_click\"].progress_apply(lambda x: x.ewm(alpha=0.9).mean().shift())\n",
    "\n",
    "    df_combined[\"user_click_ewm2\"] = df_combined.groupby(\"user_id\")[\"is_click\"].progress_apply(lambda x: x.ewm(alpha=0.7).mean().shift())\n",
    "\n",
    "    df_combined[\"dummy_ones\"] = 1\n",
    "    df_combined[\"n_ad_seen\"] = df_combined.groupby(\"user_id\")[\"dummy_ones\"].cumsum()\n",
    "    del df_combined[\"dummy_ones\"]\n",
    "\n",
    "    df_combined[\"prev_add_seen_days_ago\"] = ((df_combined[\"impression_time\"]-df_combined[\"impression_time\"].shift()).dt.seconds)/60\n",
    "\n",
    "    user_same = df_combined[\"user_id\"]==df_combined[\"user_id\"].shift()\n",
    "    df_combined.loc[~user_same, \"prev_add_seen_days_ago\"] = np.nan\n",
    "\n",
    "    df_combined[\"prev_click_sum\"] = df_combined.groupby(\"user_id\")[\"is_click\"].progress_apply(lambda x: x.expanding().sum().shift())\n",
    "\n",
    "    df_combined[\"time_since_ads_started\"] = ((df_combined[\"impression_time\"]-df_combined[\"user_id\"].map(df_combined.groupby(\"user_id\")[\"impression_time\"].min())).dt.total_seconds())/60\n",
    "\n",
    "    df_combined = df_combined.sort_values([\"app_code\", \"impression_time\"]).reset_index(drop=True)\n",
    "\n",
    "    df_combined[\"dummy_ones\"] = 1\n",
    "    df_combined[\"app_code_count_n\"] = df_combined.groupby(\"app_code\")[\"dummy_ones\"].cumsum()\n",
    "    del df_combined[\"dummy_ones\"]\n",
    "\n",
    "    df_combined[\"app_click_ewm\"] = df_combined.groupby(\"app_code\")[\"is_click\"].progress_apply(lambda x: x.ewm(alpha=0.6).mean().shift())\n",
    "\n",
    "    df_combined[\"prev_app_click\"] = df_combined.groupby(\"app_code\")[\"is_click\"].progress_apply(lambda x: x.expanding().mean().shift())\n",
    "\n",
    "    df_combined[\"prev_app_click_sum\"] = df_combined.groupby(\"app_code\")[\"is_click\"].progress_apply(lambda x: x.expanding().sum().shift())\n",
    "\n",
    "    df_combined = df_combined.sort_values([\"impression_time\"]).reset_index(drop=True)\n",
    "\n",
    "    df_combined[\"prev_user_app_click_mean\"] = df_combined.groupby([\"user_id\", \"app_code\"])[\"is_click\"].progress_apply(lambda x: x.expanding().mean().shift())\n",
    "    \n",
    "    df_train = df_combined.loc[df_combined[\"impression_id\"].isin(train_ids)].reset_index(drop=True)\n",
    "    df_test = df_combined.loc[df_combined[\"impression_id\"].isin(test_ids)].reset_index(drop=True)\n",
    "    return df_train ,df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a8971160dc54694b80608635c80cbcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74723), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c02773b87264205a160f4deddba6b0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74723), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ceb9d48108174fd48d6361e1fbe56140",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74723), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "560570145642447dab96eda54588fb96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=490), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d9de50123424e58880f5f265ea203f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=490), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a57f3cac0fa453ea0474f3023a095d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=490), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b94b5739a25a4aa3aa8be5793b04df46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=82788), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df_f_tr, df_f_te = generate_combined_features(df_f_tr, df_f_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa524b1e960b49a6bcedd7c8fbcd9c81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=89157), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea936d70ae3a450295977b00b1c1f9a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=89157), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28932f54b3c34e20a6d14119d6e692f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=89157), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a64ee227d76b4206b0014e8c630cd742",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=522), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f67e8d55b5b4b9b89309b43986cd64a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=522), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cf5ddb7433a4d628461413974c1a435",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=522), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2c58faaef2043fc8cde1302f51957b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=99679), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df_f, df_ft = generate_combined_features(df_f, df_ft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_count_feats(df_train, df_test):\n",
    "    df_train[\"user_count_leak\"] = df_train[\"user_id\"].map(df_train[\"user_id\"].value_counts())\n",
    "    df_test[\"user_count_leak\"] = df_test[\"user_id\"].map(df_train[\"user_id\"].value_counts())\n",
    "\n",
    "    df_train[\"app_count_leak\"] = df_train[\"app_code\"].map(df_train[\"app_code\"].value_counts())\n",
    "    df_test[\"app_count_leak\"] = df_test[\"app_code\"].map(df_train[\"app_code\"].value_counts())\n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f_tr, df_f_te = generate_count_feats(df_f_tr, df_f_te)\n",
    "df_f, df_ft = generate_count_feats(df_f, df_ft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictors = df_f_tr.columns.drop([\"impression_id\", \"impression_time\",\"is_click\", \"server_time\",\n",
    "                                    \"os_version\", \"user_id\", \"app_code\", \"session_number\", \"item_number\", \"time_since_ads_started\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['is_4G', 'visit_number', 'impression_time_hour', 'impression_time_dom',\n",
       "       'user_click_ewm', 'user_click_ewm2', 'n_ad_seen',\n",
       "       'prev_add_seen_days_ago', 'prev_click_sum', 'app_code_count_n',\n",
       "       'app_click_ewm', 'prev_app_click', 'prev_app_click_sum',\n",
       "       'prev_user_app_click_mean', 'user_count_leak', 'app_count_leak'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predictors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb = lightgbm.LGBMClassifier(n_estimators=650, max_depth=5, learning_rate=0.005, colsample_bytree=0.6, random_state=2019)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.6,\n",
       "        importance_type='split', learning_rate=0.005, max_depth=5,\n",
       "        min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "        n_estimators=650, n_jobs=-1, num_leaves=31, objective=None,\n",
       "        random_state=2019, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb.fit(df_f_tr[predictors], df_f_tr[\"is_click\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = lgb.predict_proba(df_f_te[predictors])[:,1]\n",
    "preds_tr = lgb.predict_proba(df_f_tr[predictors])[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7491758398634748, 0.7435771493923812)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.roc_auc_score(df_f_tr[\"is_click\"], preds_tr), metrics.roc_auc_score(df_f_te[\"is_click\"], preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a2a5b3eb8>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAEWCAYAAACdcmi1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXmcjuX+x99fYydboUFmyGDMmLEvv0NIgyhSEklj6VRaThFykqXOOZElS7ZTxESRJZFKqjGRyBZThDpMWcoyM9LMGGb5/v647+f2PM88z8wkMnK9X6/7Nfd9rd/reob5Ptd13d+PqCoGg8FgMBgMhgsUutIGGAwGg8FgMBQ0jINkMBgMBoPB4IVxkAwGg8FgMBi8MA6SwWAwGAwGgxfGQTIYDAaDwWDwwjhIBoPBYDAYDF4YB8lgMFzTiMgcERl1pe0wGAwFCzFxkAwGw8UgIglAZSDLLbm2qh77A222BRaparU/Zt3ViYgsAI6o6vNX2haD4VrHrCAZDIY/wp2qWtrtumjn6FIgIoWvZP9/BBEJuNI2GAyGCxgHyWAwXHJEpIWIfCkip0Vkt70y5MrrLyLfichvInJQRB6x00sBHwFVRCTFvqqIyAIR+bdb/bYicsTtOUFEnhWReCBVRArb9VaIyEkROSQi/8jFVqd9V9siMlxETojIzyJyl4h0FpEDIpIkIs+51R0rIstF5B17PDtFJNItP1RE4ux52CMiXb36nS0iH4pIKjAQ6AMMt8f+vl1uhIj8z25/r4h0d2ujn4h8ISKTRCTZHuvtbvkVRGS+iByz899zy7tDRHbZtn0pIhH5/oANhmsA4yAZDIZLiohUBT4A/g1UAIYCK0Skol3kBHAHUAboD0wRkUaqmgrcDhy7iBWp3kAXoByQDbwP7AaqAu2Bp0WkYz7buhEobtcdDbwOPAA0BloDo0Wkplv5bsAye6xvA++JSBERKWLbsQ6oBDwJvCUiddzq3g/8B7gOeBN4C5hgj/1Ou8z/7H7LAi8Ai0Qk0K2N5sB+4AZgAjBPRMTOWwiUBMJsG6YAiEgj4A3gEeB64L/AahEpls85Mhj+8hgHyWAw/BHes1cgTrutTjwAfKiqH6pqtqp+AmwHOgOo6geq+j+1+BzLgWj9B+2YrqqHVfUs0BSoqKovqup5VT2I5eT0ymdbGcB/VDUDWILleExT1d9UdQ+wB3Bfbdmhqsvt8q9gOVct7Ks0MN62IxZYg+XMuVilqpvseUr3ZYyqLlPVY3aZd4DvgWZuRX5U1ddVNQuIAQKByrYTdTvwqKomq2qGPd8Afwf+q6pfqWqWqsYA52ybDQYDcNXu1xsMhgLBXar6qVdaEHCviNzpllYEWA9gbwGNAWpjfUkrCXzzB+047NV/FRE57ZYWAGzMZ1uJtrMBcNb+edwt/yyW45Ojb1XNtrf/qrjyVDXbreyPWCtTvuz2iYg8CAwBgu2k0lhOm4tf3PpPsxePSmOtaCWparKPZoOAaBF50i2tqJvdBsM1j3GQDAbDpeYwsFBV/+6dYW/hrAAexFo9ybBXnlxbQr5eq03FcqJc3OijjHu9w8AhVQ25GOMvgptcNyJSCKgGuLYGbxKRQm5OUnXggFtd7/F6PItIENbqV3tgs6pmicguLsxXbhwGKohIOVU97SPvP6r6n3y0YzBck5gtNoPBcKlZBNwpIh1FJEBEituHn6thrVIUA04CmfZqUge3useB60WkrFvaLqCzfeD4RuDpPPrfCpyxD26XsG0IF5Gml2yEnjQWkbvtN+iextqq2gJ8heXcDbfPJLUF7sTatvPHccD9fFMpLKfpJFgH3IHw/Bilqj9jHXqfJSLlbRtusbNfBx4VkeZiUUpEuojIdfkcs8Hwl8c4SAaD4ZKiqoexDi4/h/WH/TAwDCikqr8B/wCWAslYh5RXu9XdBywGDtrnmqpgHTTeDSRgnVd6J4/+s7AckQbAIeAUMBfrkPPlYBVwH9Z4+gJ32+d9zgNdsc4BnQJmAQ/aY/THPKCe60yXqu4FJgObsZyn+sCm32FbX6wzVfuwDsc/DaCq27HOIc2w7f4B6Pc72jUY/vKYQJEGg8FwkYjIWKCWqj5wpW0xGAyXFrOCZDAYDAaDweCFcZAMBoPBYDAYvDBbbAaDwWAwGAxemBUkg8FgMBgMBi9MHCRDgaBcuXJaq1atK21GgSc1NZVSpUpdaTMKPGae8oeZp/xRkOdpx44dp1S1Yt4lDb8X4yAZCgSVK1dm+/btV9qMAk9cXBxt27a90mYUeMw85Q8zT/mjIM+TiPx4pW34q2K22AwGg8FgMBi8MA6SwWAwGAwGgxfGQTIYDAaDwWDwwjhIBoPBYDAYDF4YB8lgMBgMBoPBC+MgGQwGg8FgMHhhHCSDwWAwGAwGL4yDZDAYDAbDH+Tw4cO0a9eO0NBQwsLCmDZtGgBJSUlERUUREhJCVFQUycnJgBVbqWzZsjRo0IAGDRrw4osvOm2dPn2aHj16ULduXUJDQ9m8ebNHX5MmTUJEOHXqFAAiMkxEdtnXtyKSJSIV/NkqIq+KSIrb8xAR2Ssi8SLymYgE2ent3NrdJSLpInKXnfeWiOy3+3tDRIrY6W1F5Fe3OqPd+hksInvsOotFpHhubbnVa2qPqYdbWrSIfG9f0W7pRUXkNRE5ICL7ROQeO32Km00HROR0Xp+pcZAMBoPBYPiDFC5cmMmTJ/Pdd9+xZcsWZs6cyd69exk/fjzt27fn+++/p3379owfP96p07p1a3bt2sWuXbsYPdrxI3jqqafo1KkT+/btY/fu3YSGhjp5hw8f5pNPPqF69epOmqpOVNUGqtoA+Cfwuaom+bJTRJoA5bySvwaaqGoEsByYYLe73q3dW4E0YJ1d5y2gLlAfKAE85NbeRlc9VX3R7rcq8A+7n3AgAOiVV1siEgC8DHzsllYBGAM0B5oBY0SkvJ09EjihqrWBesDn9lgGu43lVeBdX/PjjnGQDPlCRMaKyFD7/kURuS2Xsv1EZMafZ53BYDBcWQIDA2nUqBEA1113HaGhoRw9epRVq1YRHW0tcERHR/Pee+/l2s6ZM2fYsGEDAwcOBKBo0aKUK3fBnxk8eDATJkxARPw10RtY7CvDdjYmAsPd021HKM1+3AJU81G9B/CRq5yqfqg2wFY/dbwpDJQQkcJASeBYPtp6ElgBnHBL6wh8oqpJqpoMfAJ0svMGAOPsdrNV9ZQPO/zOkbexhj8JEQlQ1awrbccfRVVH513q93E2I4vgER9c6mb/cjxTP5N+Zp7yxMxT/jDzlD/ymqeE8V08nxMS+Prrr2nevDnHjx8nMDAQsJyoEycu/J3fvHkzkZGRVKlShUmTJhEWFsbBgwepWLEi/fv3Z/fu3TRu3Jhp06ZRqlQpVq9eTdWqVYmMjPRph4iUxHIUnvBj6hPAalX9ORcHayDwkY/0XsArPvosAvQFnnJLbikiu7EcoKGqukdVj4rIJOAn4CywTlXX5daWverUHWv1qqlb0arAYbfnI0BVEXF5kv8SkbbA/4AnVPW4Wx9BQA0g1t8EuDAO0iVCRIKBtcBXQEPgAPAgsBd4A+gAzBCRbcBMoCLWcuXfgZ+B3UBNVc22f8n3288ZPvr6O/AwUBT4AeirqmkisgBIB8KAysAQVV0jIv2wfsmKYf1ivK2qL+QylgeBoYAC8ara1yt/AbBGVZeLSFNgGlAKOAe09yrbBXgeuNPbkxeRh+1xcMMNFRldP9OfSQabyiWs/6wNuWPmKX+Yecofec1TXFycc3/27FmeeuopHnroIXbu3ElmZqZHvus5NTWVRYsWUaJECbZs2ULHjh1ZtGgR+/fvZ8eOHfTr149+/frx6quvMmjQIO6//36effZZJk6cSFxcHOnp6WzatMnblDuBTb6210SkCnAv0NbfOETkAaAJ0MYrPRBr++tjH9VmARtUdaP9vBMIUtUUEekMvAeE2Ftg3bD+Bp0GlonIA6q6KJe2pgLPqmqWl0Pny7tTLJ+mGtYcDBGRIcAkLKfLRS9geb4WK1TVXJfgAoLtD+hv9vMbWE5GAjDcrdxnQIh93xyIte9XAe3s+/uAubn0db3b/b+BJ+37BVhOWiEgBMurLg70w3LCrsfa3/0Wax/YV9thWM7ZDfZzBfvnWKxvAq5+emA5aAeBpnZ6GfsXtB8wA8sp2wiUz2v+ateurYa8Wb9+/ZU24arAzFP+MPOUP/I7T+fPn9cOHTro5MmTnbTatWvrsWPHVFX12LFj6u//uqCgID158qT+/PPPGhQU5KRv2LBBO3furPHx8VqxYkUNCgrSoKAgDQgI0JtuukmBXXrh/++VwP3q+//2LsAv9t+kBCAb+MEt/zbgO6CSj7pPAa/5SB+D5QAV8tWnXSYBuAHLOZvnlv4gMCu3toBDbvamYG2z3YW1RfZft3L/tdMESHW1AdwE7PGy52vg//zZ636ZM0iXlsOq6nLpFwGt7Pt3AESkNPB/WJ7zLqwPNdCtzH32fS9XHT+Ei8hGEfkG6IPl1LhYqta+6/dYzktdO/0TVU1U1bNYh9Na4ZtbsbzrUwDq56CfTR3gZ1XdZpc9o6qur1ntgGeBLmrtERsMBsNfFlVl4MCBhIaGMmTIECe9a9euxMTEABATE0O3bt0A+OWXX1x/sNm6dSvZ2dlcf/313Hjjjdx0003s378fgM8++4x69epRv359Tpw4QUJCAgkJCVSrVo2dO3cCZAKISFmslZ9Vfuz7QFVvVNVgVQ0G0lS1ll23Idbfo66qesJH9RxndkTkIayzQL1VNdst/Uaxl3tEpBnWF/ZErK21FiJS0s5vj+WQ+W1LVWu42bsceExV38NayeogIuXtlakOwMdqTej7XFgla4+1i+OyrQ5QHvB8LdAPZovt0qJ+nlPtn4WA02qdovdmNTDOPp3fmNz3RxcAd6nqbnv7rG0+bPCX7o3kkvd7yh4EagK1ge35bM9gMBiuSjZt2sTChQupX78+DRpY/8W/9NJLjBgxgp49ezJv3jyqV6/OsmXLAFi+fDmzZ8+mcOHClChRgiVLljgHr1999VX69OnD+fPnqVmzJvPnz8+PCd2xzvWkuieKyIfAQ6p6LJe6E4HSWF/eAX5S1a52/WCslZjPverMAX4ENtt13lXrjbUewCARycQ6a9TLdly+EpHlWFtwmVgrOa/l0ZZPVDVJRP4FbLOTXnT7Mv8ssFBEpgIngf5uVXsDS9TlmeZFfpaZzJX3xYUttpb28+vAM9jLi27lvgTute8FiHTLWwYsxG3Z0U9fp4BKQBGs0/sL7PQFwIdYjtjNeG6xHQMqYG2xxZP7FtsB7G08ft8W23V4brHVwfLew/KaP7PFlj/Mlkj+MPOUP8w85Y+CPE/Adi0AfwP/ipfZYru0fAdEi0g8ljMy20eZPsBA+4T/HqxDay7eAR4g9+01gFFYh8E/AfZ55e3H8vQ/Ah5V1XQ7/Qss52sXsEJVfa7qqOoe4D/A57aNOd5acCt7Hmtb8FW77CdYDpkrf7893mUicnMeYzIYDAaDocBgttguLdmq+qhXWrD7g6oe4kK8BrzyluP7dL53udn4dr7AOr0/2Ef6CVX19+qnd/sxQIxX2li3+35u99uAFl5NLLAvVPVrrGBdBoPBYDBcNZgVJIPBYDAYDAYvzArSJUJVE4DwS9mmiMwE/uaVPE1VfZ7Yc1/Z8UpfgL2i49b29VghB7xpr6qJv9dWg8FgMBj+SpgVpAKMqj6uF/RsXFe+XmfIR9uJPtpuYJyjq4/g4GDnzZkmTZoAMGzYMOrWrUtERATdu3fn9OkLuozjxo2jVq1a1KlTh48//jjXdgB27dpFixYtnPStW7f6tKNTp06UK1eOO+64wyN9xowZ1KpVy0NcE2DixImOUGd4eDgBAQEkJSX5Ff3Mjy3btm0jICCAzz+/8MJNTEwMISEhhISEOK9bA7zzzjtEREQQFhbG8OEXlBcWLFhAxYoVHdvmzp3r9N2yZUvCwsKIiIjgnXcuHBUcOHAgkZGRRERE0KNHD1JSHB1Qg8FwtXKlT4mby1yq5i22/OLrbRpXgDl3Pv74Y83IyFBV1eHDh+vw4cNVVXXPnj0aERGh6enpevDgQa1Zs6ZmZmb6bUdVNSoqSj/80JJK+uCDD7RNmzY+bfv000919erV2qVLF4/0nTt36qFDh/y2r6q6evVqbdeunapawfR27NihqqpnzpzRkJAQ3bNnT562ZGZmart27fT222/XsWPHqqpqYmKi1qhRQxMTEzUpKUlr1KihSUlJeurUKb3pppv0xIkTqqr64IMP6qeffqqqqvPnz9fHH388h4379+/XAwcOqKrq0aNH9cYbb9Tk5GRVVf3111+dcoMHD9Zx48b5HGdBoyC/nVWQKMjzhHmL7bJdZgXpGkREgkXk/jzKtBWRNRfZ/kXXNVwaOnToQOHC1g56ixYtOHLkCACrVq2iV69eFCtWjBo1alCrVi2/K0IuRIQzZ84A8Ouvv1KlShWf5dq3b891112XI71hw4YEBwfn2sfixYvp3bs34F/0My9bXn31Ve655x4qVarkpH388cdERUVRoUIFypcvT1RUFGvXruXgwYPUrl2bihUrAnDbbbexYsWKXG2sXbs2ISEhAFSpUoVKlSpx8uRJAMqUKQNYXzjPnj2bm5CowWC4SjBnkK5NgoH7gbevsB0ORqw2f7iLZroEMkWEDh06ICI88sgjPPzwwx513njjDe67zwrSfvToUVq0uPDSYbVq1TycD1/tTJ06lY4dOzJ06FCys7P58ssvL+mY0tLSWLt2LTNmzMiR5y76mZstR48eZeXKlcTGxrJt2zan/tGjR7nppptyjLdTp07s27fPiUj83nvvcf78eafcihUr2LBhA7Vr12bKlCkebYAV+fj8+fPcfPOF6BX9+/fnww8/pF69ekyePPnSTI7BYLhiGAfpDyAi72FFGC2OdXj6NRFJwQrZ3g5IxooielJE4rBiEDXD0iwboKo+v7rbkiSvYokGKvCCqq4Qkd7Ac1ihAD5Q1Wft8imqWtq+7wHcoar9bFHZM3Y7N2Jpwi0HxgOhttxJjKpOyWOcpWx76mP9zoxV1VV2hNWFWEK1YKkmf+lVtylWtNR7VPWgV54Rq/2duItmugQwJ06cyA033EBycjJDhw7l7Nmzjtr3okWLOH36NFWrViUuLo4jR47w3XffOXV//vln9uzZww033OC3nenTpzNw4EDatGnD+vXrufvuu/06ALt27SIxMdFDnNOFS1yzbNmyHumxsbHUrVuX+Ph4j3Rv0U/Ary1jx47lvvvuY+PGjfzyyy8EBgYSFxfHDz/8QEZGhmPPoUOHKF68OLt37+axxx7j9ttvp1ChQoSFhXH69Gni4uIoX748MTExFC1alNWrV9OtWzdeeeVCOLDExEQGDx7MiBEj2LBhg5MeHR3NAw88wPTp03nhhRe4/fbb8/ehXkFSUlJ8flYGT8w8XaNc6T2+q/niQpRplwDs9VgOTR87fTQww76PA163728Bvs2l3ZeBqW7P5YEqWFo2FbGclFgsuRGAFLeyPfCMrL0M6zB+PWxhQixpkjV5jM0pA7wEPGDfl8OKtF0KKAkUt9NDsPfCXXWxdOd2ANXzmktzBil/5HUWYsyYMTpx4kRVVV2wYIG2aNFCU1NTnfyXXnpJX3rpJee5Q4cO+uWXX+baTpkyZTQ7O1tVVbOzs/W6667L1T7vM0gu/J1Buuuuu/Stt97ySPMl+pmbLcHBwY6IZ6lSpbRcuXK6cuVKffvtt/Xhhx926j/88MP69ttv57Dhv//9rw4bNixHemZmppYpU8Z5/vXXX7Vhw4a6dOlSf1OgcXFxfuegoFGQz9YUJAryPGHOIF22y5xB+mP8w44gvQVrJSkESyHZ9XqLu2At2GJ/qroBKCMi5fy0exsw0/WglthrUyBOVU+qJQj7FpajlRfvqSVeuxeonO+RedIBGGGvOMVhrZhVx5I6ed0WzV2GZ0DIUKyVoztV9aeL7NeQB6mpqfz222/O/bp16wgPD2ft2rW8/PLLrF69mpIlSzrlu3btypIlSzh37hyHDh3i+++/p1mzZn7bAeu8jeutsNjYWOcczqXg119/5fPPP3cEPMG/6Gduthw6dMgR8ezRowdPP/00d911Fx07dmTdunUkJyeTnJzMunXr6NixIwAnTlianMnJycyaNYuHHnoIsFbVXKxevZrQ0FAAzp8/T/fu3XnwwQe59957Pez94YcfnPv333+funXrYjAYrm7MFttFIiJtsRyZlqqaZm+hFfdRVP3c+3p2mveRl9upT/ey3jacy2cbuSFYW2T7PRJFxgLHgUisVap0t+yfbVsaYunAGS4Dx48fp3v37gBkZmZy//3306lTJ2rVqsW5c+eIiooCrIPac+bMISwsjJ49e1KvXj0KFy7MzJkzCQgI8NsOwOuvv85TTz1FZmYmxYsX57XXLH3J7du3M2fOHOc1+NatW7Nv3z5SUlKoVq0a8+bNo2PHjkyfPp0JEybwyy+/EBERQefOnZ06K1eupEOHDpQqVcoZkz/Rz86dO/u1xR8VKlRg1KhRNG3aFIDRo0dToUIFAJ566il2797tpNeuXRuwtvFWr15N4cKFqVChAgsWLABg6dKlbNiwgcTERCdtwYIFREREEB0dzZkzZ1BVIiMjmT3bX6B7g8Fw1XCll7Cu1gtLQ+19+74ulnPQFstZ6WWnPw+8at/HAXPs+1bAN7m0PZ6cW2yBWGrHNwABwKdANzv/B6wVm0LACjy32Hq4tZNi/2wMfJ7H+NriucU2AxD7uaH9cwrwjH3f3/p18thiqwzsBtrmNZ9miy1/FOSl/oKEmaf8YeYpfxTkecJssV22y2yxXTxrgcK2MO2/sLbZAFKBMBHZAdwKvOhWJ1lEvgTmAANzafvfQHkR+dbewmunqj8D/wTWYzkdO1V1lV1+BJZDEou1cpMX8UCmiOwWEV+6bd78C2s7LV5EvrWfAWZhifNuAWpjjd1BVY8DdwIzRaR5PvoxGAwGg6FAYLbYLhJVPQfkeE1FRFDVUcAoH9VWqOo/89F2ChDtI/1tfLyar9abact9pPfzei5t/8wA2udhQxzWqheqehZ4xEeZ74EIt6R/+qj7ExCWW18Gg8FgMBQ0zAqSwWAwGAwGgxdmBekS41ql8ZHe1jtNRPoDT3klb1LVxy+DaT4RkY5YYQXcOaSq3f8sGwwGg8FgKGiYFaQriKrO15xisX+ac2Tb8LEPG4xzVEDJysqiYcOGjiDsZ599RqNGjWjQoAGtWrVyXjffsGEDjRo1onDhwixf7rn7+uyzzxIeHk54eLiH4OqhQ4do3rw5ISEh3HfffR6Rpd3xJ3Y7bdo0wsPDCQsLY+rUqU76fffd5wi/BgcHO2+mJSYm0q5dO0qXLs0TTzzhlE9LS6NLly7UrVuXsLAwRowYkcOG5cuXIyJs377dSYuPj3fEZAcMGEB6erpHna5duzqhCwCSkpKIiooiJCSEqKgokpOTnby4uDgaNGhAWFgYbdq0cdKnTJlCWFgY4eHh9O7dO0cfBoPhL8SVPiVuLv8Xbm+hAXOBermUHQsMvdI227a8BezHCp75BlAkrzrmLbb8MWjQIO3du7cTiDAkJET37t2rqqozZ87U6OhoVVU9dOiQ7t69W/v27avLli1z6q9Zs0Zvu+02zcjI0JSUFG3cuLEjtHrvvffq4sWLVVX1kUce0VmzZuXo35/Y7TfffKNhYWGampqqGRkZ2r59e0fY1Z0hQ4boCy+8oKqqKSkpunHjRp09e7aHOGxqaqrGxsaqquq5c+e0VatWjkCtqiVg27p1a23evLlu27ZNVVUzMjK0fv36umvXLlVVfe+99xwRXlXVFStWaO/evTUsLMxJGzZsmCMqO27cOEfQNzk5WUNDQ/XHH39UVdXjx4+rquqRI0c0ODhY09LSnPmaP3++z8/paqEgv51VkCjI84R5i+2yXWaL7U9ERAqrFeTxd6OqD11qey4jbwEP2PdvAw8BuQaGMVps/nFprh05coQtW7YwadIkR/rCn3irSxy2UCHPReK9e/fSpk0bChcuTOHChYmMjGTt2rXce++9xMbG8vbb1jsA0dHRjB07lkGDBnnU9yd2e+TIEVq0aOEEpWzTpg0rV65k+PDhTl1VZenSpcTGxgJQqlQpj1UvFyVLlqRdu3YAFC1alEaNGjliuwCjRo1i+PDhTJo0yUlbt24dERERjsRK2bJlCQgIACyZiFdeeYXXXnuNnj17eozFJR8RHR1N27Ztefnll3n77be5++67qV69OoCH+G1mZiZnz56lSJEipKWl+RXuNRgMVz9miy0XbNX7b92eh4rIWBH5h4jsFZF4EVli55USkTdEZJuIfC0i3ez0fiKyTETeB9bl0tdwEfnGfvV+vI/8OBFpYt93EpGddtnPfJT9u4h8JCIl/PR1s4isFZEdIrJRROqKSICIHBSLciKSLSK32OU3ikgte+wxIrJORBJE5G4RmWDbvVZEigCoqvN1H9gKVPsd027ww9NPP80jjzzi4fTMnTuXzp07U61aNRYuXOhzO8qdyMhIPvroI9LS0jh16hTr16/n8OHDJCYmUq5cOQoXtr4zuYvYuuNP/DU8PNwJopiWlsaHH37I4cOHPepu3LiRypUr/65I3KdPn+b999+nfXvrpcuvv/6aw4cPO1uMLg4cOICI0LFjRxo1asTixYudvFGjRvHMM894RBQHK8hmYGAgAIGBgU5k7QMHDpCcnEzbtm1p3Lgxb775JgBVq1Zl6NChVK9encDAQMqWLUuHDh3yPRaDwXB1YVaQLo4RQA1VPecmFzISiFXVAXbaVhH51M5rCUSoapKvxkTkduAuoLlaUbkr+OtYRCoCrwO3qOoh77Ii8gSWNMhdaoUi8MVrwKOq+r0dn2iWqt4qIgew5EJqYGmotRaRr4BqqvqDiADcjCXEWw/YjBVhe7iIrAS6AO+52VIE6EvOg+iufCNWmw/i4uLYvHkzGRkZVK1a1UMQdvTo0fzrX/+iXr16LFmyhN69ezNs2DCn7i+//OKI0YK1IhMaGkpERATlypWjZs2aHDp0iC+++IKzZ886KyrhTtAmAAAgAElEQVQnTpwgLS0th0BnbmK33bp1o2XLlpQoUYKgoCB++eUXj/pTpkyhWbNmOdrct28fR48ezZGelZXFc889R+fOnfnpp59ISEhgyJAhjBgxgri4OE6fPs2OHTtISUlh//79fPrpp8yZM4dixYrx9NNPM3nyZMqWLctXX31Ft27d2LJlC6mpqU4/mZmZHn26nn/88Uf279/P5MmTOX/+PI8//jgiQrly5YiJiWHRokWULl2asWPHMnLkSCda+dWIEWHNH2aerlGu9B5fQb6AYNxEZYGhWGd91mLFHXoAKG3nbcc6c7PLvn7Cim7dD5ifRz+Tgb/7SF/AhTNIcUATrMCLb/koOxYrgOQH5HLmBygNnHWzcxfwnZ03EhgETADuBj7Civq91K2PkfZ9ISwZE1d07ReBp736eh23iOC5XeYMUu6MGDFCq1atqpUrV9bKlStriRIltHPnzlqzZk2nzI8//qihoaEe9aKjoz3OIHnTu3dv/eCDDzQ7O1uvv/56zcjIUFXVL7/8Ujt06JCjfH7Fbv/5z3/qzJkzneeMjAytVKmSHj58OEfZ+fPne5xBctG/f3998sknnefTp0/r9ddf74jSFitWTAMDA3Xbtm26ePFi5/yVq+6ECRN01qxZGhgYqEFBQVq1alUtUqSItmnTRlVVa9eurceOHVNV1WPHjqnrd3DcuHE6ZswYp60BAwbo0qVLdenSpTpgwAAnPSYmRgcNGpTD7quJgny2piBRkOcJcwbpsl1miy13MvHchnTpnHXBEpNtDOwQkcJc0CtzvQlWXVW/s8t7RJj2gS/ttYsp+y2WU5fbllYh4LR6vrUWaudtBFoDzYAPgXJYsiEb3OqfA1DVbCDD/gcKlkivsyIpImOAioCn2qjhohg3bhxHjhxhyZIlLFmyhFtvvZVVq1bx66+/cuDAAQA++eQTR1jVH1lZWSQmJgLWW1/x8fF06NABEaFdu3bOG28xMTEeArIu/IndwgXx159++ol3332X3r17O/U+/fRT6tatS7Vq+dttff755/n111893oYrW7Ysp06dckRpW7RowerVq2nSpAkdO3YkPj6etLQ0MjMz2b17N/Xq1WPQoEEcO3aMhIQEvvjiC2rXru2sBHTt2pWYmJgc4+3WrRsbN24kMzOTtLQ0vvrqK0JDQ6levTpbtmwhLS0NVeWzzz7Lc74NBsPVi9liy53jQCURuR5IAe7AOkd0k6quF5EvgPuxVmU+Bp4UkSdVVUWkoap+nc9+1gGjReRttbfY1M92HNa21kwRqaH2Fptb2a+xDkOvFpGOqppDJFZVz4jIIRG5V1WXibVvFqGqu4GvgDeBg6qaLiK7sCJo3+HdTm6IyENAR6C97UgZLgOFCxfm9ddf55577qFQoUKUL1+eN954A4Bt27bRvXt3kpOTef/99xkzZgx79uwhIyOD1q1bA1CmTBkWLVrknDt6+eWX6dWrF88//zwNGzZk4EBLDWf16tVs376dF1980a/YLcA999xDYmIiRYoUYebMmZQvX96x1bX9501wcDBnzpzh/PnzvPfee6xbt44yZcrwn//8h7p169KoUSMAnnjiCR56yP97CuXLl2fIkCE0bdoUESE8PJwuXbrkOn8jRoygZ8+ezJs3j+rVq7Ns2TIAQkND6dSpExERERQqVIiHHnrICQ/Qo0cPJ3xCw4YNefjhh/P+oAwGw1WJXFgAMPhCRP4B/AM4BBy1rzZAWazVnEWqOt4+ED0V+D87PUFV7xCRfkATVX3CV/tu/YwAHgTOAx+q6nMisgBLMHa5iMRhvca/3T6z9BLWatAJVY0SkbFYYrST7OCP44EoVT3lo68aWI5UIJbG2hJVfdHO2whstPu/H0tvrYKqZrv3YZdNUTswplf/mVjCur/ZXb7rat8fderU0f379+dWxIB1Hqlt27ZX2owCj5mn/GHmKX8U5HkSkR2q2uRK2/FXxDhIhgKBcZDyR0H+j7ogYeYpf5h5yh8FeZ6Mg3T5MGeQDAaDwWAwGLwwZ5D+RESkPrDQK/mcqja/TP3NBP7mlTxNVedfjv4MBoPBYPirYFaQ/kRU9RvNqXt2WZwju7/HffRnnKMCjLfW2sCBA4mMjCQiIoIePXpw9uxZAF555RXq1atHREQE7du358cffwRg165djh5ZRESEh9ZabGwsjRo1Ijw8nOjoaDIzc8adyq2+ty0pKSkedb310bZu3eposEVGRrJy5Uqn7IABA6hUqZKHNhrA7t27admyJfXr1+fOO+90ooRnZGQQHR1N/fr1CQ0NZdy4cQCkp6fTrFkzIiMjCQsLY8yYMXnae+7cOe677z5q1apF8+bNSUhIyNPe4OBg6tevT4MGDWjSxOxmGAzXBFc6zoC5zKVq4iC5mDx5sofWmksnTVV18ODB+ve//11VVWNjYzU1NVVVVWfNmqU9e/ZUVdX9+/c7GmhHjx7VG2+8UZOTkzUrK0urVaum+/fvV1XVUaNG6dy5c3P076++L1tcOmaqvvXRXLpsqlacoYoVKzrPn3/+ue7YscNDG01VtUmTJhoXF6eqqvPmzdPnn39eVVXfeustve+++5x2g4KC9NChQ5qdna2//fabqqqeP39emzVrpps3b9b169f7tXfmzJn6yCOPqKrq4sWLnbnLzd6goCA9efJkjvm62inI8X0KEgV5njBxkC7bZVaQrgJEZIGI9LDv54pIvVzKjhWRoX+edYZLxZEjR/jggw88XmcvU6YMYH2ROXv2LHY0c9q1a+dIZ7Ro0cLRKqtdu7Yj5VGlShUqVarEyZMnSUxMpFixYtSuXRuAqKgoVqxYkcMGf/VzswUu6KMVL17cSStZsqQTQiA9Pd2j/C233EKFCjkDxu/fv59bbrklh40iQmpqqqOFVrRoUcqUKYOIULp0acBaZcrIyHD68WfvqlWriI6OBqzX9j/77DNUNVd7DQbDtccVPYMkIgGqmnUlbfBHQbVNry7R2nxzrYvVJozvwtNPP82ECRP47bffPPL69+/Phx9+SL169Xj22Wdz1J03bx633357jvStW7dy/vx5br75ZkSEjIwMtm/fTpMmTVi+fHkOrbTc6vuyZfLkyYCnPpq7gCzAV199xYABA/jxxx9ZuHCh44D4Izw8nNWrV9OtWzeWLVvm2NijRw9WrVpFYGAgaWlpTJkyxXGwsrKyaNy4MT/88AOPP/44zZs3d4JB+rLXXU+ucOHClC1blsTERG644Qa/9oqIE1DzkUceMfGPDIZrgMu2giSW0Os+W9w0XkSWi0hJW+R0tB1k8V4/wqll7XKF7LZKishhlxiqj77chVxvEJEE+z5MRLaKyC7bhhA7/QG39P+KSICdniIiL9r6Yy399DVaLEHab0XkNTvQosuGqSLypZ3XzE4fKyILRSRWRL4Xkb/nMW9/pmhtnIhMEZENIvKdiDQVkXdtO//tVs7ffM0Wke0iskdEXnArnyAiL9i2fSMidXMbswHWrFlDpUqVaNy4cY68+fPnc+zYMUJDQ1m/fr1H3qJFi9i+fbuH/hpYGml9+/Zl/vz5FCpUCBFhyZIlDB48mGbNmnHdddfl6qx41/dlyzvvvEN2djaDBw92nA9vmjdvzp49e9i2bRvjxo0jPT0913l44403mDlzJo0bN+a3336jaNGigOWsBQQEcOzYMQ4dOsTkyZM5ePAgAAEBAezatYsjR46wdetWvv3W0ZfOYS9YK0reuFaL/Nm7adMmdu7cyUcffcTMmTPZsGFDjjYMBsNfi8u9glQHGKiqm0TkDeAxOz1dVVsB2H/YfQmn7sYKyLgeS3/sY1XN+J39P4r11tZbIlIUCBCRUOA+4G+qmiEis4A+WBGkS2Fpr43Opc0ZeiGo4kKsKNPv23mlVPX/ROQW4A3AdQI1Amhht/+1iHygPqJcy58vWgtwXlVvEZGngFVY8ilJwP9EZApQCf/zNVJVk2yH6TMRiVDVeLvdU6raSEQew9Kwy7HyJUas1mHx4qWsW7eOd999l/Pnz5OWlkZUVBQjR450ytSuXZu33nrLWR3ZsWMH06dPZ+rUqWzevNkpl5qayuDBg7n//vtJT0/3ENn817/+BViRtsuWLetTgDO3+u62vPbaa1SsWJGvv/6aFi1aAJCUlESnTp34z3/+Q506dTzqZGRkEBMT46T/8ssvHuKxLp577jkADh8+TKVKlYiLi2Pq1KnUq1ePTZs2AVCzZk1iYmJo166dR93g4GBmzpxJly5dPNp12VujRg1KlizJqlWrCAsLIysri1OnThEfH59jS83bXpekS8OGDVm8eDHZ2Vd/kHgjwpo/zDxdo1yuw01YmmA/uT3fiqX0ngAE2Wm5CafeD8yx71diRYX211ccVrRqgBuwoli72tgDPAuE2GlPAMfc+tsPjLXzMoGAPMZ1D5YkxzdYUbVHuNlwq1u5n7C0zMYCL7qlv4nltPhq+08TrXVr629un88nbnkbgAZ5zNejwE4gHjgJ9LLTE4Cq9n1z4NO8fl/MIe0LrF+/Xrt06aLZ2dn6/fffq6pqdna2PvPMM86B4p07d2rNmjWdA9Uuzp07p7feeqtOmTIlR7vHjx9XVdX09HS99dZb9bPPPstRxl99X7Y888wzOeq3adPGOaR98OBB55BzQkKCBgYGehx0PnToUI5D2i4bs7KytG/fvjpv3jxVVR0/frz269dPs7OzNSUlRUNDQ3X37t164sQJ5xB5WlqatmrVSt9//32NjY31a++MGTM8Dmnfe++9udqbkpKiZ86cUVXVlJQUbdmypX700Uc5xn41UpAPHxckCvI8YQ5pX7brcq8gea9lu55d4q2OcKqPuquBcfbKSGMgNpd+3EVlnVOiqvq2vV3WBfhYLI0wAWJU9Z8+2knXXM4diUhxLOmNJqp6WCx5jeJuRfyN1196ji5yyfs9Zb/Fcm6qYUmk5IZrdSnb7d717BLhzTFfYsmVDAWaqmqyWLIo7nPhaisLE2/rolBVoqOjOXPmDKpKZGQkDz74IADDhg0jJSWFe++9F4Dq1auzevVqli5dyoYNG0hMTGTBggUALFiwgAYNGjBx4kTWrFlDdnY2gwYN4tZbbwVg+/btzJkzh7lz5/qtHxERkcOW2bNn52r/F198wfjx4ylSpAiFChVi1qxZ3HDDDQD07t2buLg4Tp06RbVq1XjhhRcYOHAgixcvZubMmQDcfffd9O/fH4DHH3+c/v37Ex4ejqrSv39/IiIiiI+PJzo6mqysLLKzs+nZsyd33HEHsbGxfu0dOHAgffv2pVatWlSoUIElS5bkau/Bgwfp3r07AJmZmdx///106tTpUnzEBoOhIHO5PC+sFSQFWtrPrwPPYK0u3OBW7kvgXvtegEi3vGVYgRVn5dHXXGCQff80F1aQanJBTmWqnVcP+B6oZKdX4MKKVkoe/ZTDErAtgbX69S0XVlPiuLDi1Qr4xr4fi7XyUhy4HmtlqYqf9jvZ81HSZZv9cwE5V5AqAoeBGl5lx2I5Lq2wVrl89uXeln3fFkv3Da9+fM4XEIm1UlUIqGzPSz+7jPMZ223E5fX7YlaQ8kdB/iZbkDDzlD/MPOWPgjxPmBWky3Zd7tf8vwOiRSTe/sPq6ytnH2CgfeZoD9DNLe8d4AH7Z25MAgaJyJdYW2wu7gO+FUuVvi7wpqruBZ4H1tl2fYIl2ponqnoay9H7Bmu7cJtXkWTbhjnAQLf0rVhbXluAf6mP80d2+2uxVs622zb7fV1fVU9ind951567d7zyv7DrfyAiN/hoIl/4my9V3Q18jfWZvQFsutg+DAaDwWAoaFw2sVoRCcZakQjPo+hfAhGJA4aq6nav9LHYKvdXwq6rBSNWmz8KsmhmQcLMU/4w85Q/CvI8iRGrvWyYQJEGg8FgMBgMXly2w7OqmsCF19wvCfIniq+KyEqghlfys6r6sa/yqtrWT/pYH20b0VqDwWAwGAowV9UKkv6J4quq2t1HXz6do4to24jWGnLgLVTbp08f6tSpQ3h4OAMGDCAjwwoDNnHiREdUNTw8nICAAJKSkgBYu3YtderUoVatWowffyHOqKoycuRIateuTWhoKNOnT/dpQ0xMDCEhIYSEhBATE+OkL168mPr16xMREUGnTp04deqUk/fqq69Sp04dwsLCGD58uJMeHx/vCN/Wr18/R5DIrl27eojVJiUlERUVRUhICFFRUSQnJwOwb98+WrZsSbFixTwidR8+fJh27doRGhpKWFgY06ZNyzGeSZMmISKOvbnN3enTp+nRowd169YlNDTUI7aUwWC4BrnSp8TNZS5V8xabak6h2g8++ECzs7M1Oztbe/XqpbNmzcrxNs3q1au1Xbt2qqqamZmpNWvW1P/973967tw5jYiI0D179qiq6htvvKF9+/bVrKwsVb0Qb8idxMRErVGjhiYmJmpSUpLWqFFDk5KSNCMjQytWrOjEMBo2bJiOGTNGVS3R3Pbt22t6erpHuxkZGVq/fn3dtWuXqqqeOnVKMzMznb5WrFihvXv39oiDNGzYMEdQdty4cTp8+HCnza1bt+pzzz2nEydOdMofO3ZMd+zYoaqWWG5ISIgz3vXr1+tPP/2kHTp00OrVq/sUmnWfO1XVBx98UF9//XVVteJBueIr/ZUpyG9nFSQK8jxh3mK7bNdVtYL0V0NEqojI8jzKfGn/DBaR+/8cy3zakXKl+r4W8CVU27lzZ0QEEaFZs2aOIK07ixcvpnfv3oAlx1GrVi1q1qxJ0aJF6dWrF6tWrQJg9uzZjB492pENqVSpUo62Pv74Y6KioqhQoQLly5cnKiqKtWvXOv9ZpKamoqqcOXOGKlWqOO2OGDGCYsWKebS7bt06IiIiiIyMBOD6668nICAAsKISv/LKKzz//PMe/buLyEZHR/Pee+85bTZt2pQiRTyVhgIDA2nUqBEA1113HaGhoRw9etTJHzx4MBMmTPArOus+d2fOnGHDhg0MHGi9fFq0aFHKlSvns57BYLg2MAH8riBqve7fI48y/2ffBmNFBn/7Mpt1yRGRwqqaq47ItSxWm5tQLViSFwsXLmTatGlkZV2IY5qWlsbatWuZMWMG4CnCClCtWjW++uorAP73v//xzjvvsHLlSipWrMj06dMJCQnx6MdX/aNHj1KkSBFmz55N/fr1KVWqFCEhIU4wxwMHDrBx40ZGjhxJ8eLFmTRpEk2bNuXAgQOICB07duTkyZP06tXL2X4bNWoUzzzzDCVLlvTo//jx4wQGWhE3AgMDOXHiRP7nMCGBr7/+mubNrZ3qTZs2UbVqVcdB88Z77g4ePEjFihXp378/u3fvpnHjxkybNo1SpUrl2waDwfDXwjhIfxIi8jLwo6rOsp/HAr8B/VU1XETCgPlAUayzYfeopU+XoqqlgfFAqB0fKUZVp/joox/QFSgJ3AysVNXhdp6rHUSkB3CHqvazI2CfxYoTFQT0B6KxxHq/UtV+bu1PBtoByViyIidF5GZgJlbgyjQsqZR9drtJQEMsOZJnfNhrtNiAcePGkZGRwW+//cauXbtITEz00H2aNGkSNWvWJCsry0MTKjY2lrp16xIfb8nfffvtt/z8889O/nfffcexY8eIi4sjLS2No0ePMmnSJDZs2MA999yT4xzSDz/8QEZGhlP/0KFDFC9enE8//ZSXXnqJ2bNnU6VKFaZPn87DDz9M3759+fXXX/nmm28YP348+/bto2vXrrz99tvs37+fTz/9lDlz5lCsWDGeeeYZAgICKFu2LF999RXdunVjy5YtHlpsmZmZHuP2fk5ISKBEiRI5NLHOnj3LU089xUMPPcTOnTtJT0/nzTffZPLkycTFxZGens6mTZsoW7asU8d77vbv38+OHTvo168f/fr149VXX2XQoEEMGDDg4j7UqwSjMZY/zDxdmxgH6c9jCVY071n2c08sLbP+9nMOYV2v+iOw4izdkUc/DbCcknPAfhF5VVUP51GnPJYWW1cs4d2/YYnLbhORBqq6C0tod6eqPiMio4ExWDptr+ElNmy3BVAbuE39yLeo6mt2farXrKWTv7k2fx17yxnnj3N6ejpnzpxh7ty5LFq0iBdeeIHChQuzdOlSChUq5BGPZdq0aTzxxBPOc7Fixdi8ebPzvHnzZpo2bUrbtm0JCgpi+PDhBAcH06ZNGyZPnpwjrovLuXKlL168mNatW1O2bFnKly9Pnz59AAgICGD8+PG0bduWOnXq8I9//IO2bdvSrl07Jk2aRHh4OMePH+fs2bN062bFfd22bRvZ2dlkZGSQkJBAv379yMzM5MSJE4wdO5a4uDiqVq1KnTp1CAwM5Oeff6ZKlSoeNsbFxVG6dGmPtIyMDO644w4effRRhgwZAsA333zDiRMneOKJJwA4deoUTz75JFu3buXGG2/0OXd169Zl3LhxPPbYYznG+FemIMf3KUiYebpGudKHoK6lCyuyeBUsmY5NWNtm39p5OYR17fQU+2db3KRA/LTfD3jd7fkjoJV7O/Z9D2CBfb8A6GPf1wS+dyvnCOtia6q5ldtF7mLDC4Do/M6NOaRt4RKqVVV9/fXXtWXLlpqWluaRr6p6+vRpLV++vKakpDh5GRkZWqNGDT148KBzSPvbb79VVdVnn33WEX5dv369NmnSJEffiYmJGhwcrElJSZqUlKTBwcGamJioR48e1RtvvFFPnDihqqrPP/+8DhkyRFVVZ8+eraNGjVJV1f3792u1atU0Oztbk5KStGHDhpqamqoZGRnavn17XbNmjUd/3mK1Q4cO9TikPWzYMI/yY8aM8TiknZ2drX379tWnnnrK5zy6CAoK8jik7WvuVFVbtWql+/btc/oaOnRojnb/ahTkw8cFiYI8T5hD2pftuja/sl85lmM5JzdirSg5qA9hXVXNTaDXH+6Cs+5Cse4h091FZd3r+BOs9YWSu9gwXBAlNlwEjz76KEFBQbRs2RKwxFtvueUWAFauXEmHDh08zsgULlyYGTNm0LFjR7KyshgwYABhYWEAjBgxgj59+jBlyhRKly7N3LlzAU+h2goVKjBq1CiaNm0KwOjRo6lQoQIAY8aM4ZZbbqFIkSIEBQU5QrYDBgxgwIABhIeHU7RoUWJiYhARypcvz5AhQ2jatCkiQufOnenSpUuu4x0xYgQ9e/Zk3rx5VK9enWXLlgHwyy+/0KRJE86cOUOhQoWYOnUqe/fuJT4+noULF1K/fn0aNLB+BV966SU6d+6caz++5g6scAV9+vTh/Pnz1KxZk/nzTSQMg+Ga5kp7aNfSBYRhidEewNJ/C+bCClIOYV373rWC1Bj4PI/2+wEz3J7XAG3t+x+AUCynZgWeK0guIVzHHh95inXuCCxttlfte59iw+5183OZFaT8UZC/yRYkzDzlDzNP+aMgzxNmBemyXeY1/z8RVd0DXAccVdWfvbJzCOt65ccDmSKyW0QGX0T3I7AcpljAu+/8kAqEicgOrDNGL9rpuYkNGwwGg8FwVWK22P5kVLW+230CthyLqo4DxvkoX9r+mQG0z6PtBVgrN67nO9zul2Nt8XnX6efLHh95pe3bUV71DwGdcmvXYDAYDIarDbOCZDAYDAaDweCFWUG6ChGRjsDLXsmHVLX7lbDHYDAYDIa/GmYF6SpEVT/WnOKzxjkq4KSnp9OsWTMiIyMJCwtjzJgxALRu3doRUK1SpQp33XUXAMnJyXTv3p2IiAiaNWvGt99+69Get7itO08++SSlS5fOke7i9wrJuvAWf42Li6Ns2bKO/S+++GKuY81tvP5EaS+mrS+++IKIiAgaNGhAkyZN+OKLL5w6P/30Ex06dCA0NJR69eqRkJDgd54MBsO1i1lBMhj+JIoVK0ZsbCylS5cmIyODVq1acfvtt7Nx40anzD333OMEV3zppZdo0KABK1euZN++fTz++OOMGnXhCNi0adMIDQ3lzJkzHv1s376d06dP+7UjMzOTBx54gIULFxIZGUliYqKHztm7777r07k6fPgwn3zyCdWrV/dIb926NWvWrMnXWFu0aOF3vBUqVGD69OmOBtsfaatx48aMHDkSESE+Pp6ePXuyb98+AB588EFGjhxJVFQUKSkpjj6dwWAwuGP+Z7gKEZEFtlwIIjJXROrlUnasiAz986wz+ENEHMcjIyODjIwMDyHV3377jdjYWGcVZO/evbRvb53Lr1u3LgkJCSQlJQG+xW3BWlUaNmwYEyZM8GvHxQjJQt7ir79nrL7G60+U9mLaKlGihFMmNTXVud+7dy+ZmZlERUUBULp06RyacAaDwQBmBalAkB8xV3+o6kN5lyr4/NXFahPGW0ESs7KyaNy4MT/88AOPP/64I64KVgDD9u3bU6ZMGQAiIyN59913adWqFVu3buXHH3/k5MmTAH7FbWfMmEHXrl0d0VdfXIyQ7OrVq/2Kv27evJnIyEiqVKnCpEmTnOCUuY3V13hz42LaWrlyJf/85z85ceIEH3zwgTP2cuXKcffdd3Po0CFuu+02xo8f7ziIBoPB4MI4SBeBiARjyX6E289DsWQ3krA01TKBvaraS0RKAa8C9bHme6yqrrKFZbtgRbUuxQX9Mu++hgN9saJaf6SqI7zy47A02raLSCfgJSwdt1Oq2t6r7N+Bu4G7VfWsj75yCM8C39vXzUBZe4xtVXWDiGzE0pJ7AKiBFfyyNjAEaAHcDhwF7rTDFHj3d82I1boLXU6dOpWUlBRGjRpF3bp1qVGjBgAzZ86kc+fOTtm//e1vzJgxg1q1alGzZk1q1arF+fPn/Yrbnjp1irlz5zJ16lTi4uLIysryKbD5e4Vk09PTefbZZ5k4cWIO8dfU1FQWLVpEiRIl2LJlCx07dmTRokV5jtXXeF34E6X9PW2lpKRQvnx55syZw+7du3niiSeYPHkyu3fvJi4ujtdee43KlSvzwgsvMGLEiDyjfP9VMSKs+cPM0zXKlY5UeTVe5PScmK4AACAASURBVIw4PRQYCxwDitlp5eyfLwEPuNKwomiXwop6fQSokEs/t2NFqi5pP1ewfy7gQoTrOKAJllNzGKjhVXasbd8TwGqXfX76+wxbBw5oDsTa92uxooDfAWwDRgLFsN6cc/XxBVAES2cuDbjdzluJreeW23UtRtIeO3asoy126tQprVChgp49e9Zn2ezsbA0KCtI1a9boiBEjtGrVqhoUFKSVK1fWEiVKaJ8+fXTNmjVauXJlDQoK0qCgIBURvfnmm3O0tXjxYo2OjnaeX3zxRZ0wYYLOmjVLAwMDNSgoSKtWrapFihTRNm3aaHx8vFasWNFpNyAgQG+66Sb9+eefc7TtrXvma6x5jddbc+1i2vKOfBwcHKwnT57UzZs3a5s2bZz0N998Ux977DG/ff3VKcgRogsSBXmeMJG0L9tlziBdWuKBt0TkAaxVJIAOwAg7QnYc1oqR65TrJ6qalEt7twHzVTUNII+yLYANagVu9C7bF8vZukf1/9k77/Cqiq0PvxMC0kQ6SguEkpBDCkiRqyIQaYpYQAERQlWveBW9UiwIVlBBEUG4V6RY6EixfNQYQFBpIkgJooQugRBKEiBtfX/MPptzknNODlwCAed9nv1k9uzZa2ZPUBYza9ZPznt6WSlVEvgHMNca63/QK0IAa4Dm1jUSuANojHaWnPyf6FWibegVrCVW/Ta0Q/m359ixY3bw9NmzZ1mxYgWhoaEAzJ07lw4dOlC06AWZvJMnT5Keng7A5MmTad68OSVKlGDkyJEcPHiQhIQEZs2aRatWrfjiiy+49957+euvv0hISCAhIYHixYuzZ8+eXONo27YtW7duJS0tjczMTFatWkVYWBj//Oc/OXz4MAkJCfzwww/UrVuXuLg4wsPDSUxMtO1WrVqVzZs3c/PNN/PXX385nWvWr19PdnY25cqV8/mt3r73UubNm61Dhw7Z49q8eTPp6emUK1eOxo0bk5ycbG9VxsbGEhbmNYTPYDD8jTFbbJdGJu4B7s7/M9+LdiI6AsOUUg60PlknEYl3NaCUakreYq4Kd5HZS237GxAFVAX2emnjS3h2DXrrsDLwKjAIaAGsdmlzHkBEspVSGeL828m34O3fiiNHjhATE0NWVhbZ2dk88sgj9hH9WbNmMXSo2+4pO3fupGfPnhQqVIiwsDA+/fRTfv3110vqe/HixWzcuJHXX3/9koRkvTFv3jwmTpxIYGAgxYoVY9asWSilfH6rt+/1Jkp7KbZWr17N22+/TeHChSlWrBizZ89GKUWhQoUYPXo00dHRiAi33nor/fv3v6RvNxgM1zlXewnrWrzQW0nHgXLoraaf0NpkNVyeH0Vvqb0NjOeCEG0D62cvXIRlvfTTjsu3xXYHejWnso/+vAnP3gAkcGHLbaLVV6RrHy52UlzKbs+8XX/HLbZLoSAv9RckzDz5h5kn/yjI84TZYsu3y2yxXQKit5JeB35GC8DuQm8rfaGU2gb8AnwgIieBN9AO01al1G/Wvb/9LEHHDW20tr28HtcXkWPogOevLOHY2Tme/2C9/61SqrwXMx6FZ0Vvyx1AO4KgV5RuRDtcBoPBYDBcdzhXNQyGq0pISIjEx8fn3fBvTlxcHC1atLjawyjwmHnyDzNP/lGQ50kptUlEGl3tcVyPmBUkg8FgMBgMhhyY4NkCgFIqHPg8R/V5EWnqqf1l6G8CcHuO6g9FZGp+9GcwGAwGw7WGWUEqAIjINsktPpsvzpHV3wAP/Rnn6H/Am6CqM9Gjq8ArwKlTp7jvvvvs9lOnXpj+QoUK2QKsHTt2tOvHjx9P9+7dc9nKyfTp06lTpw516tRh+vTpAKSlpXHvvfcSGhqKw+FwO/V1/vx5unTpQu3atWnatKkt3rp8+XJuvfVWwsPDufXWW4mNjQW0rIdzfFFRUZQvX56BAwf6tPXll1+6vRMQEMCWLVsAmD17NhERETgcDjujN8CkSZMIDw8nKiqKO+64gx07dgA6nYDTTmRkJAsWLAB0Akxnfb9+/ShVqhRjx469iN+iwWAwuHC1o8TNZS6Ra/8UW3Z2tpw5c0ZERNLT06VJkyby448/yubNm2Xv3r25Eii+9dZbMnjwYBERSUxMlDJlysj58+dFRKREiRIe+9i8ebPMnDnTazJGEZGkpCSpWbOmJCUlyYkTJ6RmzZpy4sQJSU1NldjYWBEROX/+vNxxxx3y3XffiYjIhAkT5IknnhARnUTykUcesfs7dOiQiIhs27ZNKleu7LHPhg0byqpVq3zacmXr1q1Ss2ZNEdFJHqtVqyaJiYkiItKzZ09ZsWKFiIicOnXKfmfRokXStm1bERFJTU2VjIwMERE5fPiwVKhQwb53smLFCqlUqZIkJCR4HLNBU5BPZxUkCvI8YU6x5dtlVpAKIEqpXkqp8Vd7HDlRStVQSj16tcdREPEmqNqgQQNq1Kjhsf2ZM2cQEVJSUihbtiyBgb53vBs0aMDNN9/ss83SpUtp3bo1ZcuWpUyZMrRu3ZolS5ZQvHhxWrZsCUCRIkVo2LAhBw8eBGDRokXExMQA0LlzZ1auXImI0KBBAypXrgyAw+Hg3LlznD/vnmf0999/JzExkTvvvNOnLVdmzpxJt27dAPjzzz+pW7cuFSpUAODuu+9m/vz5AG66aq6Cs8WLF7fn6ty5cx7Fczdv3kytWrUICgryOV8Gg8HgDRODZLgYagCPAjMut+FrWazWHyHanDz99NN07NiRypUrc+bMGWbPnk1AgP73yrlz52jUqBGBgYEMHTrUVqj3h0OHDlGtWjX7vmrVqhw6dMitzcmTJ/n666959tlnc70TGBjITTfdRFJSEuXLX8gGMX/+fBo0aMANN9zgZmvmzJl06dLFdlL8sTV79mwWLVoEQO3atdm1a5edoXvhwoV29nDQGmvvv/8+6enp9hYfwM8//0yfPn3Yt28fn3/+eS7nMjY21nbCDAaD4VK4aAdJKVUGqCYiW/NhPFccpdRCoBo6G/aHIvJfpVQKWmqjJZAMdBWRY5Yw7BagCVAK6CMi673YLYkWqW2EznD9mojMV0p1A15CJ2L8VkSGWO17Ay8CR9B6beet+grAJC7IkwwUkbWXqc8UESlplTsDHUSkl1JqGnDasnMzMFhE5gGjgHpWTqbpIvKBhzH0QmcSL44WuF0gIoNztrPaXhditf4I0boKvAKsWrWK8uXLM2PGDA4fPky/fv2YPHkyJUqUYNasWZQvX57Dhw/z5JNPkpqaSpUqVQAtmpnTlit79uwhIyPDHtPevXspWrSofZ+VlcVLL73EPffcw/79+9m/fz8pKSn8+OOP9ipOTvt79+7llVde4d13380l2DllyhRefPFFN5FYX7Z27NiBiHD8+HH7naeeeor27dsTEBCAw+Hg5MmT9jOHw8Gnn37KihUrePrpp3nxxRftvidMmMC+fft46aWXKFGiBEWKFAH0Ct7atWvp37+/ERjNAyPC6h9mnv6m+LMPh87WXAooC+wHNgHvX+39wctxcSHjdDG0JEc5tHPR3ap/FSvjtTUPn1jl5rgI1nqw+w4w1uW+DFqqYz8663UgEAs8gNY8c9YXAda69DkDuMMqVwd2Xo4+reeuGa87A9Os8jRgLjqIPwzYY9W3AL7JYz57AX8CN6Gdzn1oh9rn7+Faj0HKSU5B1ZxxQ/fcc4+sXr3avm/ZsqX8/PPPuezExMTI3Llz7fvvv//eZwzSjBkz5PHHH7fvH3/8cZkxY4Z937t3b/nXv/7l9k6bNm1k3bp1IiKSkZEh5cqVk+zsbBEROXDggNSpU0d++OGHXH1t2bJF6tSp47ctEZGBAwfKW2+95XHsIiL/+c9/ZNCgQbnqs7KypFSpUh7fadGihWzYsMG+X7hwoTRq1MhrH4YLFOTYmoJEQZ4nTAxSvl3+xiDdJCKngYfQ4qm3ooVUrweesTJH/4ReSaqD1g9zZqL+Ai3T4WQmgIisBkoppUp7sXs3MMF5IyLJaIHXOBE5JiKZwJdoR6upS3067lmw7wbGW6s2i60+b7wMfebFQhHJFpEdQCU/2ruyUkROicg5YAdw3QeC5CWompPq1auzcuVKAI4ePUp8fDzBwcEkJyfbcT7Hjx9n7dq1FyWm2rZtW5YtW0ZycjLJycksW7aMtm3bAvDKK69w6tSpXCe7OnbsaJ92mzdvHq1atUIpxcmTJ7n33nsZOXIkt9+eMyuEeyxRXrYAsrOzmTt3Ll27dnV7JzExEYDk5GQ+/vhj+vXrB+j4JifffvstderUAfSKVmamXm3ct28f8fHxbnFeM2fOpFWrVn7PmcFgMHjCXwcpUCl1C/AIWlrjukAp1QLtVDQTkUi0RIgneXHxUvZ0b5v38Cx3NGnedgKs8TmP41cRkTP50GfO73aNxvVlwxOu72bxN4h1O3LkCC1btiQiIoLGjRvTunVrOnTowLhx46hatSoHDx4kIiLC/st/2LBhrFu3jvDwcKKjo3nnnXcoX748O3fupFGjRkRGRtKyZUuGDh1qO0jjxo3j4YcfzmVr48aNdrls2bIMGzaMxo0b07hxY1599VXKli3LwYMHeeutt9ixYwcNGzYkKiqKyZMnA9C3b1+SkpKoXbs277//PqNGjQJ0WoE9e/bwxhtv2Mfnnc4MwJw5c3I5SN5sgRaQrVq1KsHBwW7vPPvss4SFhXH77bczdOhQ6tata/fvcDiIiori/ffftx2vH374gcjISKKionjwwQf5+OOP7RintLQ0li9fbgeNGwwGwyXjzzIT8DCwFZho3QcD86/28tf/eqG1xr62yqHAOfQ2kqDjjgBeAT6yynHAJKt8B7DNh+1R5N7uugW95VQerd22whqDs74cWrdtDe5bbINc7ERdjj6t53uAemgnbD7uW2ydXeykWD9vBVblMae9cBHhRTvULfL6XVxvW2z5RUFe6i9ImHnyDzNP/lGQ5wmzxZZvl18rSCIyV0QiROSf1v2fItLJn3cLOEvQq2Nb0SKyTjHWVMChlNoEtEIL0zpJVkqtQwdO9/Vh+02gjFLqN2sLr6WIHEEHYn8P/ApsFpFFVv0I4Ee0A7PZxc4zQCOl1Fal1A7gycvRp9V+KNqBiUUHh+fFViBTKfWrUuo5P9obDAaDwXBN4tfWh1KqLjARqCQi9ZVSEUBHEXkzX0eXz4hWqW+fs14phYgMA4Z5eG2+iLzooT6n7RQgxkP9DDwckxedyTpXNmsROQ50yau/S+xzHjDPQ32vHPclrZ8ZQHQeY5iGXoFy3nfwZ+wGg8FgMBQk/I1B+gS9CpEBIPqIf1efbxgMBoPBYDBco/gbPFtcRNbnyFh7bSat8QPniomH+hY566z8Rc/mqF4rIgPyYWhXrU8PY2iLTivgyl4RefBKjcFgMBgMhvzC3xWk40qpWlinnqykgv7ErFz3iMhUyS38mq+OytXo08MYlnoYwzXjHPXp04eKFStSv359u65Lly72aa0aNWoQFRUFQHp6Or179yY8PJzIyEi3hHEzZ84kPDyciIgI2rVrl0tEdvTo0T7FZT0Jy/qyO23aNKpUqWKP87vvvgMgKSmJli1bUrJkSZ5++mm3PjZt2kR4eDi1a9fmmWeecQbPM2LECI+2ALZu3UqzZs1wOByEh4dz7tw5n4K3oE+1hYWF4XA4ePTRC4o0F/uNBoPBUCDwJ5IbfWptBZAGHAJ+AIKudoS5uS7+wiU5ZEG6rvQptlWrVsmmTZvE4XB4fP7888/La6+9JiIi48ePl169eomIyNGjR6Vhw4aSlZUlGRkZUqFCBTtp46BBg2T48OG2jf3790ubNm2kevXqHhM7ehOW9WU3JibGLQGlk5SUFFmzZo1MnDhRBgwY4PascePGsm7dOsnOzpZ27drZIrXDhw/3aCsjI0PCw8Nly5YtIqIFZTMzM30K3u7evVuioqLkxIkT9jxd6jdeDgryqaOChJkn/yjI84Q5xZZvV55bbEqpAKCRiNytlCoBBIj3PDwGwyVxpbTYnLppzZs3JyEhwWMbEWHOnDm29teOHTuIjtax6RUrVqR06dJs3LiRBg0aICKkpqZSrlw5Tp8+Te3atW07zz33HO+++y7333+/x35chWUBW1i2c+fOPu16okSJEtxxxx3s2bPHrf7IkSOcPn2aZs2aAdCzZ08WLlxI+/a5zibYLFu2jIiICCIjIwEoV64cgE/B208++YQBAwZQpkwZe54u9zcaDAbDlSTPLTYRyQaetsqpxjm6ciilaiildiqlPlFKbVdKLVNKFfPStr9SaoN1BH++Uqq4VV9TKfWj9eyNPPq7RSm1Wim1xUoVcKdV38aysVkpNdfSfEMpdatSapVSapNSaqmVTBSlVJxS6h2l1Hql1G6nnWuFNWvWUKlSJTtzc2RkJIsWLSIzM5O9e/eyadMmDhw4QOHChZk4cSLh4eFUrlyZHTt20LevzvywePFiqlSpYjsZnvAmLOvLLugEihEREfTp04fk5GSf33Lo0CGqVq2aqw9ftnbv3o1SirZt29KwYUPefffdXHadgrdOx3H37t3s3r2b22+/ndtuu40lS5b8T99oMBgMVxt/g7SXK6VeQEtgpDorReREvozK4EodoJuI9FdKzQE6oeVPcvKViHwCoJR6E52j6SPgQ3SCz8+UUnnFKT0KLBWRt5RShYDiSqny6GSZd4tIqlJqCPC8UmqkZf9+0UK+XYC3gD6WrUARaaKUugcYjgdpmqshVusaP/TXX3+RmpqaS4Tygw8+oEmTJnZ9rVq1WL58OaGhoVSqVInQ0FB27tzJihUrePvtt5k4cSKVK1dm3LhxPP744zz88MMMGTKE9957j7i4OK/ist6EZb3Z7dGjB9HR0fTo0QOlFFOmTOHRRx9lyJAhts1du3Zx6NAh2+auXbtITk6277du3cqJEyeIi4sjIiKCTz/9NJet+Ph4VqxYwaRJk7jhhhv497//TaFChbj11lsBz4K3R48eJSkpiddee41jx47Ro0cPpk6deknfeDkw4qL+YebJP8w8/U3xZx8O2Ovh+vNq7w9e7xdQA/jd5X4I8IqXtnehM3Bvs34/zozfSUBhq1wKHzFIaI22PeiklVFWXQfgOLDFunYAnwL1gdMu9duAZdY7ccDtVrkSltitr+tqZNLeu3dvrhikjIwMqVixohw4cMDre82aNZPt27fL+vXrpVWrVnb9qlWrpH379rJ161apUKGCBAUFSVBQkBQqVEiqVasmR44ccbPjTVjWm10R91gIT+OfOnWqWwzS4cOHJSQkxGufnmzNnDlTYmJi7Gevv/66vPvuu/a9J8HbJ554QqZOnWrft2rVStavX39J33g5KMgxIwUJM0/+UZDnCRODlG+Xv5m0a3q4gvN+03AZ8FfXbBrwtIiEA6/hrq3mTefNDdECvM3RgfifK6V6onXYlsuFk2phItLXqt/uUh8uIm08jPua0mJzisy6bkulpaWRmqoXTpcvX05gYCBhYWFUqVKFHTt2cOzYMftZvXr1CA8PJzExkYSEBBISEqhatSqbN2/m5ptvduvLm7CsN7ugT6s5WbBggdspPE/ccsst3Hjjjfz000+ICJ999pkdE3XkyIWDqK622rZty9atW0lLSyMzM5NVq1bZenDeBG8feOABvv/+e0CL7O7evZvg4OBL+kaDwWAoEPjjRQE9PV1X27u73i/0CtJvLvcvACO8tD0OVERruS3ngq7aYuAxq/xPfK8gBaG3xgAGAmOBCsB+oLZVXxyoCxRBrzY1s+oLAw6rHIcO7AetAZeQ17de6RWkrl27ys033yyBgYFSpUoVmTx5sojoU2ITJ050a7t3716pW7euhIaGSnR0tCQkJNjPJk6cKKGhoRIeHi4dOnSQ48eP5+orKCjIPq21YcMG6du3r/3s008/lVq1akmtWrVkypQpedpt3bq11K9fX8LDw+W+++6Tw4cPu/VTpkwZKVGihFSpUkW2b99u9+lwOCQ4OFgGDBgg2dnZIiLy2GOPebX1+eefS1hYmDgcDhk0aJCIiBw4cEAACQ0NlcjISImMjJRPPvlERESys7Plueeek3r16kn9+vVl5syZl/yNl4OC/C/+goSZJ/8oyPOEWUHKt0vp+fWNUuojl9uiaLmJzSLS2Q8fzHCJKKVqAN+ISH3r/gWgpIiM8ND2n8BgtDDtNuBGEemllKqJlhkJRAvSviJeEmEqpWKAQeiM6SloJ3ivUqoVOinkDVbTV0RksVIqChgH3GTZHysinyil4oAXRGSjFcO0UURq+PrWkJAQiY+P929i/sbExcXRokWLqz2MAo+ZJ/8w8+QfBXmelFKbRKTR1R7H9YhfWx8i8i/Xe6XUTcDn+TIig42IJKBjfZz3o320nYjWy8tZvxdo5lI1yoeN6cB0D/WxQGMP9VvQW3I561u4lI+jV8IMBoPBYLhm8DeTdk7S0KerDAaDwWAwGK47/FpBUkp9zYVA3wAgDJibX4MyeEcpNQG4PUf1hyIy9SJshJN7BfC8iDT9X8dnMBgMBsP1gL8rSKOBMdY1EmguIkN8v2LID0RkgOTWQPPbObJsbPNg47p2jjxpr3nTIsvIyCAmJobw8HDq1avHyJEjATh37hxNmjQhMjISh8PB8OHDbVu9evWiZs2atq0tW7Z4HIc3XTJvemk59eH69esH+NaHmz17NhERETgcDgYPHmzXv//++4SFhREREUF0dDT79u0DYMuWLbbuWkREBLNnz7bf6du3L5GRkURERNC5c2dSUlIAOH/+PF26dKF27do0bdrUzkruSxOuRYsWhISE2N+TmJjoc1wGg8FwVfEnkht4x586c5nrUq/8PsXmSXvNmxbZl19+KV26dBERkdTUVAkKCpK9e/dKdna2nDlzRkRE0tPTpUmTJvLjjz+KiD79NnfuXJ9j8KZLJuJdL82V559/3taE86YPd/z4calWrZokJiaKiEjPnj1lxYoVIiISGxsrqampIiLy8ccfyyOPPCIiIvHx8bJ7924RETl06JDcfPPNkpycLCIip06dsvt/7rnnZOTIkSIiMmHCBHniiSdEROdNctrypQl31113yYYNG3J9l7dx/S8U5FNHBQkzT/5RkOcJc4ot3y5/V5Bae6jzLuZkyFeUUgnW6TCUUuvyaBunlPrbn3Bo3ry5rQeWF0opUlNTyczM5OzZsxQpUoRSpUqhlKJkSX0AMCMjg4yMDJRSfo/BVZesTJkyti6Zq16aUsrWS3NFROvDOaU9vOnD/fnnn9StW5cKFSoAcPfddzN//nwAWrZsSfHixQG47bbbbB21unXr2rIqlStXpmLFinZ+olKlStn9nz171v7eRYsWERMTA0Dnzp1ZuXIlImJrwhUt6pqGyzfexmUwGAxXE58xSNbR8aeAYKXUVpdHNwJr83NgBv8QkX9c7TFcDvJTrNYpUOuJ8ePH89lnn9GoUSPGjBlDmTJl6Ny5M4sWLeKWW24hLS2NDz74wHausrKyuPXWW9mzZw8DBgygadMLO5Mvv/wyr7/+OtHR0YwaNYobbrjBrS9vumR56aXBBX04ZzunPlzXrl05cOCArQ/XqlUrdu3aZSeoXLhwIenp6bm++9NPP/UoWLt+/XrS09OpVauWXde7d2++++47wsLCGDNmTK5vCQwM5KabbiIpKYny5ct7nWunrUKFCtGpUydeeeWVXA6mt3EZDAbDlSavIO0ZwP+h446GutSfEaPDZqOUWghUQ+eI+lBE/quUSgH+A7QEkoGuojXL4tDSHE3Q0h99RGS9F7sl0XpnjdBB8q+JyPwcbVLEymuklBoM9ACygf8TkaEu7QKAqcABEXnFS39t0Fm4bwD+AHqjA/KHishDSqn7gVnovEcBwA4RCba+6RfgVnRiyZ7Ai0A4MNtHf1dEi80Zn5NTe82bFtm2bds4fvw4M2fO5MyZMzz77LOULFmSypUrAzB27FhSUlIYNmwYoaGh1KxZk/vuu4+YmBgyMjIYM2YMTz75pL3C4sSbLlnJkiW96qU5cerDOTWhvOnDlStXjqeeeor27dsTEBCAw+Hg5MmTbraWL19ObGwsY8eOdatPSkriueeeY+jQoaxevdquj4mJ4bHHHmPcuHG89tprtG/fnpSUFH788Ud7pSqn3lxOTTiAAQMGUKFCBdLS0hg+fDhpaWm0bds2z3FdCkY7yz/MPPmHmae/Jz4dJBE5BZwCugEopSqinYCSSqmSIrI//4d4TdBHRE4opYoBG5RS84ES6GSa/1ZKvYoWbHVGrZYQkX8opZoDU3DJdZSDYcAp0fIhKKXKeBuAUqo98ADQVETSlFKu+0mBwJforNxveXnfoygt8DbQwGp2J/AbOidSIPCzi4l0EWmulHoWWIR2lk4AfyilPhCRJHIgIv8F/gtQPbi2jNmWP4okCd1b6J8JCZQoUcJjwrfg4GA6dOhAixYtmDt3LjExMdx9t9bX/frrrwkMDMz13qZNm0hKSqJ3795u9UWKFGH06NG52h85csQt4dzMmTO58847adGiBWPHjrXrjxw5Qnh4uH2fmZlJly5d2LRpE3v27LHrnVtsAP/4xz946KGHCAsLo0WLFrz00ksA/Pe//3V7Z8WKFXz11VesWrWKihUr2u+fPn2aFi1aMGbMGB5++GGP8xgYGMh7773HO++8Q0hICFWrVqVZs2ZkZmZy/vx5OnbsaK8IJSQkkJKS4jW5XmJiIhs3bsxzXJdKQU7sV5Aw8+QfZp7+nvh7zP8+4H2gMpCIlqTYCTjyb2jXFM8opR60ytXQOaKyAedxoC+Ar1zazwStfaaUKqWUKi0iJz3YvRvo6rwRkWQfY7gbmCoiaVZb1xW+/wBzvDlHFrehV4vWWn/JFQF+FJFMpdQepVQ99KrX++jkkIXQ4rhOFls/t6E12o4AKKX+RM9JLgfJlWKFCxHvYyssPzhy5Ai33HIL4K5FVr16dWJjY3nsscdIS0vjrTc+NgAAIABJREFUp59+YuDAgRw7dozChQtTunRpzp49y4oVKxgyZIibLRFh4cKFHjXS2rZty0svvURysv41Llu2jJEjR1K2bFlbL61p06Z89tln/OtfF3KzuurD7dmzB9D6cM6YH1d9ONDOR8WKFUlOTubjjz9mzpw5APzyyy888cQTLFmyxM0JSU9P58EHH6Rnz55uzpGI8Mcff1C7dm1EhK+//prQ0FAAOnbsyPTp02nWrBnz5s2jVatWPuOxMjMzOXnyJOXLlycjI4NvvvnGdkC9jctgMBiuJv7+k/1N9F+gK0SkgVKqJdaq0t8dpVQLtHPSzFq5icNdKNaJeCl7urfN+3h2MW3XAS2VUmNE5JyP95eLiKff6xp0UH4GsAItjFsIrQ3nxClOm427wG42BUCstlu3bsTFxXH8+HGqVq3Ka6+9RlxcHFu2bEEpRY0aNfjPf/4D6K2g3r17U79+fUSE3r17ExERwdatW4mJiSErK4vs7GweeeQROnToAED37t05duwYIkJUVBSTJk0CYOPGjUyaNInJkydTtmxZhg0bRuPGOin5q6++asc2TZw4kV69enH27Fnat2/vFocza9YsunVz/7UkJibStm1bAgICqFKlCp9/fiGt1bPPPsuvv/5q91G3bl0ABg0aREpKiu0EVa9encWLFzNnzhxWr15NUlIS06ZNA2DatGlEREQQExPD6dOnEREiIyOZOFEna+/bty89evSgdu3alC1bllmzZtn916hRg9OnT5Oens7ChQtZtmwZQUFBtG3bloyMDLKysrj77rvp37+/z3EZDAbDVcWfo25YxwiBX4EAq7z+ah/BKwgXcD/wtVUOBc4BLdDOSler/hXgI6scB0yyyncA23zYHoXWN3Pel7F+JgDlrXKK9bMd2hEqbt2XdemvEXq77GssMVoPfXkUpbXKLaxnb1r3P1ljUK59uLT9xsWu/czXdaXFaq9VCvJx44KEmSf/MPPkHwV5njDH/PPt8veY/0krYHgN8KVS6kMgfyJqrz2WAIHWKb830M4DQCrgUEptAloBr7u8k2wdz58E9PVh+02gjFLqN6XUr+iAb4+IyBL0NtdGpdQW3Fd3EJH3gc3A51bAds73jwG9gJnWt/yEdvhAxxpVApyRu1uBrdZ/nAaDwWAwXHf4u/VxP3AWGAh0R59iet3nG38TROQ8HnJCKaUQkWHoQOuczBeRF/2wnQLEeKiv4VIu6VIeRQ4xWnEXjh2OD8S7KO1Z9Mk25/3jPvqIQ68a5XpmMBgMBsO1gl8OkuhTTUFAHRGZrpQqjo5BMRgMBoPBYLju8PcUW390vpqyQC2gCnp7KNrXe39nXFd2ctS3yFmnlOoNPJujeq2IDMiHoaGU+hmXFSGLHiKyLT/6MxgMBoPhWsPfGKQBaAX50wAi8jtgzuNeJkRkquQWj80X58jqr6mH/q4r58iTOO2gQYMIDQ0lIiKCBx98kJMndWYFb+K08fHxtrBqVFQUpUqVYuzYsW79jB49GqUUx48f9zqW06dPU6VKFTfxVm/Crfv376dly5Y0aNCAiIgIW0AXYOTIkXTv3p2QkBCWLl3q1kdWVhYNGjSwT9UBdtv69evTp08fMjIy7GdxcXFERUXhcDi466677PoPP/yQ+vXr43A43L512LBhREREEBUVRZs2bTh8+LBb/xs2bKBQoULMmzcPgO+//95t7ooWLZpLPsVgMBgKNP5EcgM/Wz9/sX4GooN0r3qUubmuj+tyn2LzJE67dOlSycjIEBGRwYMHy+DBg0XEuzitK5mZmVKpUiVJSEiw6/bv3y9t2rSR6tWry7Fjx7yO5ZlnnpFu3bq5ibd6E27t37+/fPzxxyIisn37dgkKCrLLERERsnTpUvnzzz8lODhYMjMz7ffGjBkj3bp1k3vvvdeu+/bbbyU7O1uys7Ola9eutt3k5GSpV6+e7Nu3T0S02K2IyLZt28ThcEhqaqpkZGRIdHS0LWLrKlr74Ycf2kK1zrlp2bKltG/f3qNgb1JSkpQpU8YWpL0SFORTRwUJM0/+UZDnCXOKLd8uf1eQVimlXgKKKaVaA3PRR8avCyxZEE/105RSna/0ePxFKdVLKTX+ao+jIOJJnLZNmzYEBupdZVdRVG/itK6sXLmSWrVqERQUZNc999xzvPvuuz4TJG7atImjR4/Spk0bv8atlOL06dMAnDp1ypY3cequFSlShJo1a1K7dm3Wr9cKNQcPHuTbb7+lX79+brbuuecelFIopWjSpIn9vTNmzOChhx6ievXqAHZyxp07d3LbbbdRvHhxAgMDueuuu1iwYAGA23ykpqa6ffNHH31Ep06dvCZ5nDdvHu3bt7cFaQ0Gg+FawN9TbEPRx9G3AU8A3wGT82tQSqlCIpKVX/YNBY/LKVbrS5zWyZQpU+jSpQuAT3FaJzmTNS5evJgqVaoQGRnptY/s7Gz+/e9/8/nnn7Ny5cpczz0Jt44YMYI2bdrw0UcfkZqayooVKwAtDnvbbbfZ77oK2g4cOJB3332XM2fOeBxHRkYGn3/+OR9++CEAu3fvJiMjgxYtWthacz179qR+/fq8/PLLJCUlUaxYMb777jsaNWpk23n55Zf57LPPuOmmm/j+++/tcS1YsIDY2Fg2bNjgsf9Zs2bx/PPPe50ng8FgKIj4dJCUUtVFZL+IZAOfWNf/hFKqBjp30M9oja/daHHTHWhdsjbAeKXUBmACOoFhGtAfOIJOVhksItnWabp46z6DHLgElxcB9qADkdOUUjXRQryB1lic7RVaHLYVsBedXdrXt4wCOqJzQi0TkReUUhXQAezVrWYDRWStUqqEZTvc6neEiCxSSvWybBRHB8AvEJHBPvrsjRaCPWLN3Xmr/j50QsoiaFmP7sAxa37+IVooN8B65zZ0TqXhQBZa7625l/5qAJ+jteUAnhaRdZat8cBd1lwFAFNEZJ5SKhoYbX3nBuCfotMh5LSdL2K13sRpnXzxxRecPHmSKlWqEBcXl6c4bUZGBvPnz6dDhw7ExcVx7tw5hgwZwnvvvWffuwq1OlmwYAEhISH88ccfucRbvQm3zpkzhzvvvJNHHnmE7du306lTJ6ZMmcLBgwfZuXMnpUuXJi4ujiNHjrB9+3Z+//13MjIyOHPmDFu2bCEpKSnX944ePZrg4GCysrKIi4tj3759xMfHM2bMGNLT0xkwYABKKapVq8b9999Ps2bNKFasGEFBQfz111+2vdatW9O6dWu+/PJLXnjhBXr37s2IESPo0qULa9as4a+//mL79u2UL1/e7jspKYnNmzdTtGjRKyr2acRF/cPMk3+Yefqb4mv/DS226izPvxx7ekANdJbp2637KeikhgnAYJd2K9FpBQCaArFWeRHQ0ip3ASb76KucS/lN4F9WeTHQ0yoP4EI26oeA5egUBpWBk0BnL7bLop0PZzbp0tbPGcAdVrk6sNMqvw085myLdlRKoJMz/onOLVUU2AdU89LnLeiM1hXQjtBaYLz1rIzLWPoBY6zycLSTBtr5nG+VtwFVXMfupc/iQFGrXIcLWdU7o1cSA4CbgWSrrihwgAtZuD9z9u/rqlazlgQN+eayXE727t3rFoMkIjJt2jS57bbb3OJhnnrqKfnss8/s+969e8vs2bPt+4ULF0rr1q3t+61bt0qFChUkKChIgoKCpFChQlKtWjU5cuSIW1+PPvqoVKtWTYKCgqRcuXJy4403ypAhQyQnU6dOteOTwsLCZP/+/fazmjVrytGjR+Xtt9+Wt99+246FaNOmjaxbt06GDh0qVapUkaCgIKlUqZIUK1ZMunfvbr8/YsQIuf/++yUrK8uuGzlypAwfPty+79Onj8yZMyfXuF588UWZMGFCrvqEhAR7XmvUqGHPQ4kSJaRChQqyYMECu+3YsWOlf//+uWzkNwU5ZqQgYebJPwryPGFikPLtysuZ+cVT+X/qUDtI+13uWwELLQcpyKoriU5MucXlcjoaj3JBqmMB0NpHX3ehs39vQ69yON9LAgpb5VIuDtJYoI/L+1/5cJAC0atZn1qOVRGrPjHHuA8BNwIbgd9c6vcD9SwH6RMXu/+H5WB56PMB4DOX+2dcHKRwYJn1rfHAEqu+GpajC8wCOljlSWhnsD8ujqSHPm9CryBts8ad5jJXvXPOFRAJrHapjwa+yuvPRX5IjeR0kP7v//5P6tWrJ4mJiW7tRo0aJb169ZLs7GxJSUmRevXqya+//mo/79Kli0yZMsVrP0FBQT6DtEXcnaCMjAy7fXp6unTq1EkmTpwoIiLt2rWTqVOniojIjh075JZbbpHs7Gz57bff3IK0a9as6RakLaL/J+4apP3JJ59Is2bNJC0tza3djh07pFWrVpKRkSGpqanicDhk27ZtInIhYHvfvn0SEhIiJ06cEBGxg7VFRMaNGyedOnXK9Y0xMTG5grSbNm0qsbGxPucmPyjIf6EVJMw8+UdBnifjIOXflVcMki+B1f8Fb2KtqdbPAOCkiER5eHcxMFIpVRa4FYj10c804AER+dXaymrhYwx51bs30ir3TdAOQFfgabSzF4AWrj3r2t7avuskIvE56pviLu6ahe+tT2/j+wh4X0QWWwK6I6xxHlBKHVVKtUKvxHW36p+0+r4X2KKUihKRJA92nwOOoh2fALTWHHjffvS5LXml8CROO3LkSM6fP0/r1q0BHag9adIkr+K0AGlpaSxfvtwWss0LV3Fab5w/f96rcOuYMWPo378/H3zwAUoppk2bhlIKh8PBI488Qu/evSlZsiQTJkygUCHfuVqffPJJgoKCaNasGQAPPfQQr776KvXq1aNdu3ZEREQQEBBAv3797HQInTp1IikpicKFCzNhwgTKlCkDwNChQ4mPjycgIICgoCBbjNcXCQkJHDhwwC2NgMFgMFwz+PKe0H9ZnwbOoONsTrvcn74Uj4wLW2zNrPtPgH/jIsBq1a8DHrbKCoh0eTYXvarxcR59HUfnayqMXi2ZZtUv5sJ21z9x32Jbit5iuwVr28iL7ZJARatcFjhhlWcAg1zaRVk/30bH7Di3wRpYP3thrQJZ998ALbz0eQt6C66c9U1ruLCC9Atwq1WeCsS5vNcJOAy841JXy6X8i3OcHvr8APi3Ve6t/8gIwMPWWAPQOm0nuLDF5ip6Ow14Nq8/F0as1j8K8r9kCxJmnvzDzJN/FOR5wqwg5dvl85i/iBQSkVIicqOIBFpl530pX+/mwU4gxhJFLQtM9NCmO9DXEmndjtaDczIbeMz66Yth6GDw5cAul/pngQFWILhrZO0C4Hf0dtJEYJUP2zcC31jfsAq90gJ626uRUmqrUmoH8KRV/wbaqdmqlPrNur8oROQIemXoR2AFWnzWyQhgrlJqDdoxdGUx2qGb6lL3nlJqmzWW1ejtQk98jP5d/QTU5cIq33zgIHrb8D/oeT4lIufQjtRcpdQ2IBu9nWcwGAwGwzWDczXjynWoT0V9IyL182hquEwopRoBH4jInZfZbkkRSVFKlQPWowPv/7oUWyEhIRIfH593w785cXFxtGjR4moPo8Bj5sk/zDz5R0GeJ6XUJhFplHdLw8Xibx4kwzWKUmooehuxez6Y/0YpVRp9ou6NS3WODAaDwWAoaFxxB0lEEoDLunqklJqA1opz5UMRmeqp/SXYXwDUzFE9RESWemp/mfq8LIKyIjIKGOVHf22Bd3JU7xWRB33YbnExYzEYDAaD4VrBX6mRAo2IDJDc4quXxTmy7D/owX6+OUdWn1dUUFZElnroz6tzVNC5GLHaL7/80k1YNSAggC1btgAwc+ZMwsPDiYiIoF27drYobV7irU7atWtH6dKl3URkAe688067v8qVK/PAAw8AWl7kvvvuIzIyEofDwdSp7n+MU1NTcwnfbtq0ifDwcGrXrs0zzzzjDK6nS5cudh81atQgKkofCvUmzguwZMkSQkJCqF27NqNGXfCrvQnfJicn8+CDDxIREUGTJk347bff7Hdq1KhBeHg4UVFRbhm5QcuThISE4HA4GDzYa15Ug8FguHpc7Shxc5lL5OqK1bqydetWqVmzpojofEUVKlSwcxYNGjTITrDoS7zVlRUrVsjixYvd8hPl5KGHHpLp06eLiMhbb71ljysxMVHKlCkj58+fd2ubU/i2cePGsm7dOsnOzpZ27drJd999l6uP559/Xl577TUR8S7Om5mZKcHBwfLHH3/I+fPnJSIiQrZv3y4i3oVvX3jhBRkxYoSIiOzcuVNatWpl9+ktP1RsbKxER0fLuXPnRORC7qXLSUE+dVSQMPPkHwV5njCn2PLtui5WkAwXjyU+nFcbjyK+ftq/5HcvBxcjVuvKzJkzbc01538kqampiAinT5+25Ud8ibe6Eh0dzY033uh1nGfOnCE2NtZeQVJKcebMGUSElJQUypYta49506ZNJCcnuwnfHjlyhNOnT9OsWTOUUvTs2ZOFCxe69SEizJkzx/4ub+K869evp3bt2gQHB1OkSBG6du3KokWLAO/Ctzt27CA6OhqA0NBQEhISOHr0qNfvBZg4cSJDhw7lhhv0DrI3kVuDwWC4mpgg7auIUipQRC6PANnF8xI6N1OB4GqK1boye/Zs2ykoXLgwEydOJDw8nBIlSlCnTh0mTJhgt/Uk3nqxLFiwgOjoaNvhevrpp+nYsSOVK1fmzJkzzJ49m4CAAFv49qmnniItLc1+/9ChQ1StWtW+dxWxdbJmzRoqVapEnTp1AO/ivIcOHaJatWputn7++Wc3WzmFbyMjI/nqq6+44447WL9+Pfv27ePgwYNUqlQJpRRt2rRBKcUTTzzB448/Dmix3DVr1vDyyy9TtGhRRo8eTePGjS9p/gwGgyG/MA7SRZAzRYFS6gV0fqET6HxHmcAOEemahzjtveiEiiXQ2bc99TUY6IHOI/R/IjJUKRWFzilUHPgDLYuSrJSKA14QkY1KqfLoJdca3oRwLZHdYkqpLcB2EcnzhJtSahDwCDpwfIGIDLfqF6LlTIqiA+P/m+O98sDXwJsi8m2OZwVCrNbJjh07EBGOHz9OXFwcmZmZvP3220ycOJHKlSszbtw4Hn/8cXr06AF4Fm/1hDcRWYAJEyZwzz332M9WrVpF+fLlmTFjBocPH6Zfv35MnjyZZcuWERISQvHixdm8ebMtfLtr1y6Sk5Pt97du3cqJEyfc+vrggw9o0qSJXedNnHf37t0cOXLEbrdz504OHz7sZiun8O3tt9/O+PHj7ZWn2rVr88svv3DmzBnee+89ypcvT3JyMi+88AJnz54lMjKSU6dOsW3bNkaNGsWuXbvo2LEjM2bM8LoKdykYcVH/MPPkH2ae/qZc7T2+a+lCZwH/zeX+BXSCxsPADVadU7TWlzjtQaCsj37aozOJF7fuy1o/twJ3WeXXgbFWOQ5oZJXLAwlWuRdehHCxsofn8b3ODONtgP+iM5oHoDNoN88xtmLopJHlnO+iM2z/jA+9POd1JbTYRDyL1ToZOHCgvPXWW/b9+vXr3WJqVq1aJe3bt8/1nqt4qydyaqQ5OX78uJQtW1bOnj1r191zzz2yevVq+75ly5by888/28K3lSpVchO+PXz4sISEhNjtZ8yYIY8//rh9n5GRIRUrVpQDBw7Ydd7EedetWydt2rSx650CuU48Cd+6kp2dLUFBQW7xWU6GDx8u7733noiItG3b1i2mIzg4OJc+3v9KQY4ZKUiYefKPgjxPmBikfLtMDNLlYSvwpVLqMfQqEminYqi1ShOHdlCqW8+Wi8gJH/buBqaKSBqAiJxQSt2Edr6c2b2nA839GNtKEXFmuN4BBF3EdzlpY12/oLN3hwJ1rGfPWNnOf0KvJDnrCwMrgcEisvwS+rzsLFmyhHfeeYfFixdTvHhxt2fZ2dnMnTuXrl272nVVqlRhx44dHDt2DIDly5dTr149AH7//Xe73eLFiwkNDb3o8cydO5cOHTpQtGhRu6569eqsXLkSgKNHjxIfH09wcDBffvkl+/fvZ9asWYwePZqePXsyatQobrnlFm688UZ++uknRITPPvuM+++/kHR+xYoVhIaGum3DVa9endjYWER0fNVPP/1EaGgojRs35vfff2fv3r2kp6cza9YsOnbsCMDkyZNZunQpM2fOJCDgwv82Tp48SXp6ut2mefPmlCpVitTUVM6cOQPoGK1ly5bZJwofeOABYmO1hOLu3btJT0+nfPnyFz1/BoPBkJ+YLbaLIxP31AjOv9nuRTsrHYFhSikHerXFmzhtKr5RXJw4sOu4iuZ4djFCuL7GM1JE3BRbLVHcu9G6emnWVp+z/0xgE9AW35It+cLFiNUCrF69mqpVqxIcHGzbqFy5MsOHD6d58+YULlyYoKAgpk2bBngXb80pVnvnnXeya9cuUlJSqFq1Kp9++ilt27YFYNasWQwdOtRt3MOGDaNXr16Eh4cjIrzzzjt5Og8TJ06kV69enD17lvbt29O+fXv72axZs+zgbCe+xHnHjx9P27ZtycrKok+fPjgcDsC78O3OnTvp2bMnhQoVIiwsjE8//RTQzt2DD+osEZmZmTz66KO0a9cO0CkY+vTpQ/369SlSpAjTp0+/rNtrBoPBcFm42ktY19KFXhU5jhaLvQG9avI6UMPl+VH0lppf4rRe+mmH5y22X4E7rfIItHwIwGTgn1Z5IO5bbB6FcNFCvIXzGIfrFtvPQEnrvgpaBPh+4GurLhQ452I/BS36+xUwNK+5NWK1/lGQl/oLEmae/MPMk38U5HnCbLHl22VWkC4CEclQSr2Odhb2ogVwCwFfWFtgCu20nFRKvQGMRYvTKiAB6ODZcq5+llgB2RuVUunAd+hTZzHAJKVUcXRskTMqeDQwRynVA4j183P+a41ts+QRpC0iy5RS9YAfrX/pp6DFgpcAT1qCvfFoh9H1vSylVFfga6XUaRH52M+xGQwGg8FwVbniYrUGgyeMWK1/FGTRzIKEmSf/MPPkHwV5noxYbf5hgrQNBoPBYDAYcmC22K4iSqlw4PMc1edFpOkVHEM59GmznESLSNKVGofBYDAYDAUJs4J0FRGRbZJbIPaKOUfWGJI8jCHqWnaOPAnVzp07F4fDQUBAABs3brTr09PT6d27N+Hh4URGRrolg/MmVOtNBNYTWVlZNGjQwE2sduXKlTRs2JCoqCjuuOMO9uzZA+iTdA0bNiQwMJB58+bZ7b///nu7v379+lG0aFFbTmTv3r00bdqUOnXq0KVLF/vI/XPPPWe/U7duXUqXLp3LVlRUlJut2NhYGjZsSP369YmJiSEzU2es8CVI++GHH1K/fn0cDgdjx461670JAyckJFCsWDG7/yeffDLP36fBYDBcFa52lLi5zCVyeU+xeRKq3bFjh+zatUvuuusu2bBhg10/fvx46dWrl4ho0dSGDRtKVlaWT6FaV1xFYD0xZswY6datm1uiyDp16siOHTtERGTChAkSExMjIjqx5a+//io9evSQuXPnerS3aNEiKVOmjJ3o8uGHH5aZM2eKiMgTTzxhi8i6Mm7cOOndu3eu+qSkJNtWVlaWVK1aVeLj40VEZNiwYTJ58mQR8S5Iu23bNnE4HJKamioZGRkSHR0tu3fvFhHvwsCeknfmBwX51FFBwsyTfxTkecKcYsu3y6wgXeMopUZYkicopV5XSt3to20vpdT4Kze6q4Mnodp69eoREhKSq62r2GrFihUpXbo0GzdutP8D8SRU60TEXQQ2JwcPHuTbb7+lX79+bvVKKU6fPg3AqVOnbLs1atQgIiLCLRFjTlatWkX79u0pXrw4IkJsbCydO3cGICYmJpdQLbgL8Loyb94821ZSUhI33HADdevWBbSMyvz583PNkasg7c6dO7ntttsoXrw4gYGB3HXXXSxYsADwTxjYYDAYCjImBglQShUSkayrPY7/FRF59WqP4VK5XGK1/gjVuhIZGcmiRYvo2rUrBw4cYNOmTRw4cIAmTZr4FKqF3CKwORk4cCDvvvuunVHayeTJk7nnnnsoVqwYpUqV4qeffvL4vie+//57Xn/9dQCSkpIoXbq07Yh4Eqrdt28fe/fupVWr3JJ/s2bN4vnnnwegfPnyZGRksHHjRho1asS8efM4cOCAPUeeBGnr16/Pyy+/TFJSEsWKFeO7776jUaPch2lyCgPv3buXBg0aUKpUKd58803uvPNOv7/fYDAYrhTXvYNkCcwuQecuaoDWROuJlt2Ygk6COF4ptQGYAFQA0oD+wBF0csZgEcm28g/FW/cZHvrqjxZfLQLsAXqIzjA9DZ1E0YHWJ3teRL6xxGQfRCedrAnMEJHXfHxLT7T+mwBbRaRHjufT0GK685RSjYEP0fpv54HoHG3vBV4B7hOR4x76ehgYjs6+fUpEmlvjbSQiT1ttvgFGi0icUirFmr+70UkoXwLeRcurDBSRxR76uOxitXkJ1Z48eZJNmzaRkpICQK1atVi+fDmhoaFUqlSJ0NBQdu7cyYoVK3wK1UJuEVhXfvzxRzIyMjhz5kwusdpXX32VN954g7CwMDvT9aBBg+x3//rrL7Zv354rg3ZSUhJ//PEHRYsWJS4ujpMnT3L27FnbbmJiImlpabniqJo1a8aaNWty2dq8ebNtC2Dw4MH06dOHjIwMGjVqxLlz53wK0tauXZv777+fZs2aUaxYMYKCgvjrr7/c+s8pDJyens6MGTO46aabiI+Pp1OnTkydOpUSJUrk8Zu9OIy4qH+YefIPM09/U672Hl9+X2iBWQFut+6noJ2MBLROmLPdSqCOVW4KxFrlRUBLq9wFmOyjr3Iu5TeBf1nlaWgnLQCtVXYQLcnRC+2EleOC2GsjL7YdaOesvHXvzK49AnjBpZ/OaAftT6CxVV8K7Qz3Qmf3fhBYA5Tx8S3bgCpW2SnA2wvvmbkFaG+VFwDL0JnFI4Etef2eLncmbW+xLjljkHLSrFkz2b59e55CtZ6S80MYAAAgAElEQVREYF0ZOnSoVKlSRYKCgqRSpUpSrFgx6d69uyQmJkpwcLDdbt++fVKvXj23d2NiYjzGII0dO9Ytlik7O1vKlStnx/rkFJsVEYmKipK1a9d6tNW/f3+v87B06VJ5+OGHc9X7EqR98cUXZcKECfa9L2FgJ3n9Pi6VghwzUpAw8+QfBXmeMDFI+Xb9XWKQDojIWqv8BXCHVZ4NoJQqCfwDmGuJy/4HuMWljXN/oKvzHS/UV0qtUUptA7qjnRonc0QkW0R+RzsvTnXT5aJPkp1Fy3LcgWdaAfPEWu0R32K3IcAREdlgtT0tIs7lmZbAEOBeEUn2YWMtMM1aFSvko52TdLQTCNq5WiV6lW0b2kktkKSlpZGaqqXxli9fTmBgIGFhYT6FasGzCKwrI0eO5ODBgyQkJDBr1ixatWrFF198QZkyZTh16hS7d+/2aNcXM2fOtGOBQMcytWzZ0j7xNn36dDeh2vj4eJKTk239tJy2csYlJSYmAnD+/Hneeecd+4SZN0Fa13f279/PV199Zdv0Jgx87NgxsrL0bvaff/7J77//7qZ/ZzAYDAWF636LzSJnunDnvVM0NgA4KSKezmsvBkYqpcoCt+JbymMa8ICI/GptR7XwYwze6nNyMQK2vtr+CQQDdYGNXtogIk9awrr3Alss6RNvYr0AGda/ZgCysURyRW9NXtE/Z56EasuWLcu//vUvjh07xr333ktUVBRLly4lMTGRtm3bEhAQQJUqVfj8c52WypdQLXgWgT18+DD9+vXju+++8zq2wMBAPvnkEzp16kRAQABlypRhypQpAGzYsIEHH3yQ5ORkvv76a4YPH8727dsBfTz+wIEDREZGutl755136Nq1K6+88goNGjSgb9++9rOZM2fStWvXXEKwTlt33XWXW/17773HN998Q3Z2Nv/85z/tuCVvgrQAnTp1IikpicKFCzNhwgTKlCkDwNNPP+1RGHj16tW8+uqrBAYGUqhQISZNmpQroN5gMBgKBFd7CSu/Ly5ssTWz7j8B/o3eYivv0m4d8LBVVkCky7O56ISOH+fR13G0iGthYDkwzaqfhtZTCwBq4b7Fdhgoi95i24rvLbbdWNt4XNwW2424b7GFoGOwHD6+pZZL+RcgCr26tc76jmrAaVzEaV3a22PK+czbZcRq/aMgL/UXJMw8+YeZJ/8oyPOE2WLLt+vvssW2E4ixRFXLAhM9tOkO9FVK/QpsRyvVO5mNFmf1tb0GMAwdDL4cLWTrSjywCvg/4EkROWfV/4B2vrYA80XE46qOiGwH3gJWWWN839sgRCQdvS34kdV2OS6rPSISb33vXKVULS9m3lNKbVNK/QasRgerr0WL9G5DC+Ru9jYGg8FgMBiuZf4uW2zZIpIzZW8N1xsR2Qu08/SyiMxDryr5REQm4tn5AlgrIs95qE8U61SYH/anA9Nz1I1wKfdyKW8AbsthYpp1ISK/AGE++nrIy6PuXtqX9DSmnM8MBoPBYLgW+LusIBkMBoPBYDD4zXXvIIlIgojUz7ul/yilJiiltuS4evsYQy9rFSpn/bScq0dKqXIebG+xRGUvO0qplz309XJ+9HUluBgdtvXr19uaYJGRkXYWaNA5jhwOB/Xr16dbt26cO6d3REWEl19+mbp161KvXj3GjRuXawxbtmyhWbNmOBwOIiIimD37ws5s3759iYyMJCIigs6dO9v5mPbt20d0dDQRERG0aNHCLfP04MGDcTgcdn867OACHTt2dPtegI8++oiQkBAcDgeDBw/O83uXLFlCSEgItWvXZtSoUXa9N603gDlz5hAWFobD4eDRRx+164cMGUL9+vWpX7++27cbDAbDNcXVDoIyl7lELl+Q9sXosDk1xEREDh8+LBUqVJCMjAw5ePCg1KhRQ9LS0kRE651NnTpVRESmTJkiPXr0kKysLBHR+m05iY+PtzXJDh06JDfffLMkJyeLiLjlD3ruuedk5MiRIiLSuXNnmTZtmoiIrFy5Uh577DEREVm7dq384x//kMzMTMnMzJSwsDC3gNH58+dLt27d3L43NjZWoqOj5dy5c25j9Pa9mZmZEhwc/P/tnXmczfX+x59vZC+ZLA1jzzKLMZYuWi3XKH5pU1HJWql0UyrKJfdWVyUKLbpJliskxHXd5GKIK7fsJNnGHtmKYcYs798f3+98O3PmnJlRxhy8n4/H53G+5/P5fD/L+5xxPj6f9/f90u3bt2tKSorGxsbqpk2bvLkH0nr74YcfNC4uTo8ePZqlj7lz5+of//hHTU1N1ZMnT2rjxo0DxkzKb0LZqTaUMDvljVC2E+aknW/pot9ButgRkequIzUi0kREsm9pZK1/8vyMrGA4Gx22TA0xgOTk5CyPw6elpXH69GnS0tI4deqUp5f2/vvvM3jwYE8vrUKFCtnarVOnjic/UqlSJSpUqODFU8qMH6SqnD592uvTV++sZcuWzJ49G3BiHSUnJ3PmzBlSUlJIS0ujYsWKgBPdd8SIEfz5z3/O0v/777/PgAEDKFasWJYxBpvv//73Py9KdtGiRenUqROzZ89GNbjW24cffsgTTzzhPdaf2cd3333HzTffTJEiRShVqhQNGjTgiy++wDAM40LjUnHSLlDOl9abOk/ABY1tFMqcCy22s9VhA1i5ciU9evRg165dTJo0iSJFilC5cmWeffZZqlatSokSJYiPjyc+Ph6A7du3M23aNGbNmkX58uUZNWpUUC02cBYfZ86coVatXx8W7N69O/PmzSMqKorhw4cDjt7ZjBkzeOqpp5g1axYnTpzgyJEjNG/enJYtWxIeHo6qctttt3mBJQcNGkS/fv2yBGIE+OGHH/jqq68YOHAgxYsX58033+Taa68NOt99+/ZRpUoV7/6IiAhWrlyZo9ZbZqDL66+/nvT0dIYMGcItt9xCgwYN+Mtf/sIzzzzDqVOnWLx4MVFRQZ8FMAzDCFlsB+l34u7gfC8iE0RkvYh8JiIlRSRRRAaLyDLgHhGpJSJfiMgqN9p2PREp49Yr5LZVUkT2iMhlQfq6RkT+IyLrRGS1/yP6ItLC1UdDREqLyMfuo/rrReRuv7rlRGSFq8kWqK9wEVnq+iRtFJEb3fyTPnU6uvpviMh4EXlfRBaLyA4RuVlExonI5sw6oUjTpk3ZtGkT33zzDUOHDiU5OZljx44xe/Zsdu7cyf79+0lKSuIf//gH4ESZLl68ON9++y0PP/wwPXr0CNr2gQMH6NKlCx9//LG34wTw8ccfs3//fiIjIz0fnTfffJMlS5bQsGFDlixZQuXKlSlSpAjbtm1j8+bN7N27l3379rFmzRqWLl3K2rVr2bZtG3feeWe2ftPS0jh27Bhff/01w4YN495778XZiQ8838wyX0QkaH5mH1u3biUhIYEpU6bQq1cvjh8/Tnx8PO3ateO6666jc+fONG/e3FtgGYZhXEjYv1znhrpAT1VdLiLjgMfd/GRVvQFARBbixD/a6kaofk9VW7lxim4GFgO3AfM1gBCuy2TgNVWdJSLFcRa42c94HAbhiMzWd/svm1kgIhVxIoT/WVUXBLn/fncsr4pIYaBkkHq+lMWRROkA/BO4HugFfCMicaq61rfyuRarPVuhWn9SU1OZMGECBw4coHjx4l4U68jISKZPn05ERARhYWGe8GrZsmVZs2ZNQBHLpKQknn76ae6//35P9NWfOnXq8Pe//50aNWoA8Kc//QmA06dP88knn7BmzRqmTp1KxYoVPefyuLg4Jk+eTMmSJVmxYgVXX3016enpHD9+nLi4ON5++21KlixJzZo1WbJkCQBnzpxh9uzZXHnllQHnm5aWxrp167wxLl26FICNGzfy008/sXDhQgoXLsymTZs8cdtChQpRt25dli93FHwqVKjA1KlTqVevHtdffz3XX389AC+//HIWQd3zhYmL5g2zU94wO12iFLQT1IWecOIp7fZ53wr4HCdSdzU3rzRwGicYZGba7JbdD4xxr2cBbYL0czmwN0j/G93rFsBc93oVrviuX/0UHFHcm3OZ103ANpyo2HE++b4RszuSNVr4A+51TWCrT72JOBIsQfs7l5G08ypUu2PHDs9pOTExUcPDw/Wnn37Sr7/+WqOiojQpKUkzMjL0oYce0lGjRqmqav/+/fWjjz5SVcdxs0mTJtn6SUlJ0VatWulbb72VJT8jI0O3bt3qXffr10/79eunqqo//fST5/j94osv6qBBg1RVderUqdq6dWtNTU3VM2fOaMOGDXXOnDk5zvf999/37t+yZYtGRERoRkZG0PmmpqZqjRo1dMeOHZ6T9saNG1XVcR73ddLOFKP997//rQ899JA39oiICD18+LCmpaXp4cOHVVV13bp1Gh0d7fV5Pgllp9pQwuyUN0LZTpiTdv79vhf0AC705C5Qdvm8b+UudBJxpUyAK3DEYwPdXxrYhRPhezdQOEi9K85ygbQauCZA/SScYJN/y8PcKgEP40TOfsjNO+FT/qDfAqmj/5j8y4Klc7VA6tSpk1599dVapEgRrVy5so4dO1ZnzpyplStX1qJFi2qFChU8xfuJEydqVFSUNmjQQBs2bKizZs3y2hk8eLDWrVtXo6Oj9cEHH/SeCDt27Ji2a9dOY2JitFmzZrp27VpVVf3mm2+0Z8+eqqo6adIkLVKkiDZo0MBLa9as0fT0dL3uuus0JiZGo6Oj9f777/ee8Jo+fbpec801Wrt2be3Zs6fXX1pamj7yyCNar149jYyM1I4dO2abs/8CKSUlRR944AGNjo7Whg0b6sKFC3Od77/+9S+tXbu21qxZU1955RUvf/v27XrttddqrVq1tGPHjt64MjIy9Omnn9bIyEiNiYnxFlGnT5/WyMhIjYyM1KZNm+qaNWt+z8f5mwnlH7RQwuyUN0LZTrZAysff94IewIWeOL9ab19n7sQAxXCOvYItkF4D3va5t6z7ehIoDMwEBuTQVzWgiHvdN7MtnF2lSJzjvRmhtkC62Anlf6hDCbNT3jA75Y1QtpMtkPIvmZP2ueF8ab11Af7k9vNf4Ooc6r4ClHUdrNcBLTML1HmirhPQUkQeD3J/C2CtiKwB7gZGuvkDgLnAIuBALuM1DMMwjAsSc9I+N5wvrbetOEd4/sS45QlAgnt9EugaoI3S7usZoG0OfWXTffMZa6Co4N18rhMzx+RfZhiGYRgXAraDZBiGYRiG4YftIP1O/HdLzgUi8i7OI/K+jFTVj89lP25f9XH8n3xJUdWm57ovwzAMw7hQsB2kEERVn1DVOL90zhdHbl8bAvQV0ouj48eP07FjR+rVq0dkZCQrVqzgueeeo169esTGxnLnnXdy/PhxABYsWEDjxo2pX78+jRs3ZtGiRV47Z86c4ZFHHqFOnTrUq1ePGTNmZOtr8uTJnsBrXFwchQoVYu3atZw4cSJLfrly5ejbty8Au3fvpmXLljRs2JDY2FjmzZuXpc3du3dTunRp3nzzTQD27NlDy5YtiYyMJDo6mpEjR2ap7ys8O2bMGMCJYdS1a1fq169PZGQkQ4cOzbWt3yLaG0j8F5wo3rGxscTFxREfH8/+/fvz+OkZhmFcIBS0l7glS6pn9xTbQw89pB9++KGqOo+0Hzt2TOfPn+/F23n++ef1+eefV1XV1atX6759+1RVdcOGDVqpUiWvncGDB+vAgQNVVTU9PV1/+umnHPtdv3691qhRI2BZo0aNdMmSJaqq+vDDD3uirps2bdJq1aplqXvXXXdpx44dddiwYarqCMeuWrVKVVV/+eUXrV27ticW6y88O3PmTFVVnTx5st53332q6ojQVqtWTXfu3JljW2cr2qsaWPxXNavo7siRI/XRRx/N0Xbnm1B+6iiUMDvljVC2E/YUW76lC/aITUT+q6rXFfQ4AiEivYFTqjrxHLUXB1RS1Xnu+w5AlKq+di7aD9BfItBEVQ/nR/u/h19++YWlS5cyfvx4AIoWLUrRokU9rTSAZs2a8dlnjh95w4YNvfzo6GiSk5NJSUmhWLFijBs3ju+//x6AQoUKUa5cuRz7njJlCp07d86Wv3XrVg4dOsSNN94IOHIcv/zyCwA///yzJ3QL8Pnnn1OzZk1KlSrl5YWHhxMeHg7A5ZdfTmRkJPv27SMqKiqb8GymOKyIkJSU5InqFi1alCuuuIKwsLCgbWVquPnjq+XmL9p70003kZiYmO2eTNFdcKKG+95jGIZxMXDBLpDyY3EkIkVU9ffpXQCqOuZcjMeHOKAJMM9tfw6OVMgFgzi/oKKqGYHK8yJWm/hae3bs2EH58uXp3r0769ato3HjxowcOTLLgmPcuHHcd9992e6fMWMGDRs2pFixYt4R3KBBg0hISKBWrVq88847VKxYMWj/06ZNY/bs2dnyp0yZwn333ectEoYMGUJ8fDyjR48mKSmJ//znP4CzkHj99ddZsGCBd7yWbY6JiaxZs4amTZ1TTn/h2c6dO9OiRQs6duzI7NmzCQ8P59SpU7z11luEhYXl2FZOBBKxzY2BAwcyceJEypQpw+LFi3OtbxiGcSFxwS6QROSkqpYWkRbAX4CDOAuJmTiRn58CSuAEVtzuCqYmA9FAReAZVZ0rIt2A9kBxoBTQSkSeA+7FCcY4S1VfEpFSwKdABE6gxZdVdZqIvIajPZYGfKmqz4rIEBxJjjfd3Z8xOEEdtwM9VPWYiCQAK3HiE12Jo+X2VYB5FgX+CpQQkRuAoe68mqhqH3dep4F6OMEdu+M83t8cWKnuI/YiEu/aqZg7ju7qhAIIxpMichtwGU6Ay+9FJAwYhyMlcgp4RFXX+87X7Wsj8H9uO//G0ZlrDtyBEzU8c25npcWWkJDAli1bWLVqFd26daNbt26MHj2axx57zBON/cc//sHx48c9vbRMdu7cyZ///GfeeOMNEhIS+Pnnn9m7dy9lypRhxIgRfPrpp3Tp0oUXX3wxYN/fffcdqsrhw4ezaTKNGzeOF154wcv/9NNPufHGG7n33nvZtGkTd999N+PGjeODDz4gPj6eb7/9lsTEREqUKJGlrdOnT/PUU0/Rq1cvVq9eDTg7UBs2bOC1117j+++/56WXXqJu3bps3LiRw4cPM2XKFE6cOMFTTz1F6dKlvd2qQG1lEkyT7t1332XXrl28+OKLlCpViqJFiwLBte3atGlDmzZtmDx5Ms8++yzdu3fP8fM7n5h2Vt4wO+UNs9MlSkGf8f3WhKsJhhPQ8DgQjvPjvw/4i1v2FL9GgB4PfIHjmF4b2IuzKOrmXoe59eKBv+PEJSqEExTxJpxgiR/69F8GJyjkFpydEYAr3dchwLPu9Xpc3TOchU7meBKA4e51O+A/Ocy1G/BOoPfuvKa6470d+AWo7459Fc6isRywFCjl3tMfGJxDf4nAk+7148BY93o08JJ73QpY6z9f9/1GnDhQ1YEMoFlun2eVGrW0Wv+5OSZV1QMHDmTx6Vm6dKm2a9dOVVXHjx+vzZo106SkJPVlz549Wrt2bV22bJmXl5GRoSVLlvT0z3bv3q1RUVEajL59++qrr76aLX/t2rVau3btLHlRUVG6e/du732NGjX04MGDesMNN2i1atW0WrVqWqZMGS1btqyOHj1aVVXPnDmj8fHxOnz48CxttW3bNov/Q6VKlfTQoUP6+OOP68SJE7387t2767Rp03JsKxN/HyR/WrRokaU8mLZdJomJiTmWFwSh7DMSSpid8kYo2wnzQcq3dMHuIPnxjaoeABCR7cCXbv4GfCJIA5+qc8SzVUR24Oy6ACxQ1aPudbyb1rjvS+MsqL4C3hSR13HkPL4SkSI4u1JjReRfOIspDxEpg7NoWuJmTcCRFclkpvu6Cr/AkmfJP1VVRWQDcFBVN7j9b3LbjQCigOXuMVBRYEUubfqO7S73+gachSKqukhErnLnmBO7VPXr3CZQ4rLCbHmtfW7VuPrqq6lSpQpbtmyhbt26LFy4kKioKL744gtef/11lixZksWn5vjx47Rv356hQ4d6CvPg+PDcdtttJCQk0KpVK6+dQGRkZDB9+nRP5d6XQH5JVatWZeHChXTr1o3NmzeTnJxM+fLl+eqrXzcIhwwZQunSpenTpw+qSs+ePYmMjOSZZ57J0tYdd9zBokWLaNGiBT/88AOpqamUK1eOqlWrsmjRIh588EFOnTrF119/Td++fXNsKxg7d+6kSpUqFClShF27drFlyxaqV6+e4z1bt26ldu3aAMyZM4d69erlWN8wDONC42J5zD/F5zrD530GWY8R1e++zPdJPnkCDNVfH3m/RlU/UtUfgMY4i66hIjJYHX+lP+Bokt2Bs0P1W8adzu877vSdr78tiuDMaYHPnKJUtedvGFsgT1zFOV70/S4V97lO4hwzevRoHnjgAWJjY1m7di0vvvgiffr04cSJE7Rp04a4uDh693YCm7/zzjts27aNl19+2XuU/dChQwC8/vrrDBkyhNjYWCZNmsTw4cMB5wd/8ODBXn9Lly4lIiKCmjVrZhvLp59+mm2BNHz4cD788EMaNGhA586dGT9+fI5OzMuXL2fSpEksWrTIG2NmaIAePXqwY8cOYmJi6NSpEwMGDEBEeOKJJzh58iQxMTFce+21dO/endjY2BzbmjVrFhEREaxYsYL27dvTtq0TSH3ZsmU0aNCAuLg47rzzTt577z3PYb1z5840b96cLVu2EBERwUcffQTAgAEDiImJITY2li+//DJbaALDMIwLnoLewvqtiaxHbHN98hNw/HOylOEcRc3D+SGvRdYjNt/jq3gc36DS7vvKQAUcZfvibt4dwOc4u0sV3Lww4Kh7PYRfj9jWATf65L8VYJzlgMQc5no3MMHnvTdm8iASC5QHdgPXuPklgTo59JeIK7SL4xye4F6PAgb52HaNe/0gMNW9boSzqKruP56ckonV5o1Q3uoPJcxOecPslDdC2U7YEVu+pYvliC2vbAGW4Dhp91bVZP//2avqlyISCaxwy07iLACuAYaJSAaQCjwGXA7MFpHiOLsrTwfosyswRkRKAjtwnKjPlsXAABFZi+OkfVao6k+uM/oUESnmZv8Z+OEsmxoCfOyK5Z7iV623GcBD7vi++Q3tGoZhGEZIkelcfNHjPu01Vx2xVSPEqFu3rm7ZsqWghxHyJCQk0KJFi4IeRshjdsobZqe8Ecp2EpFVqtqkoMdxMXKx+CAZhmEYhmGcMy6ZIzZ14wGFMiLSFnjdL3unqt6ZT/3NAmr4ZfdX1fn50Z9hGIZhXCjYDlIIoarzNbtwbL4sjtz+7gzQX0gvjtLT02nYsCH/939OHMpFixbRqFEjYmJi6Nq1K2lpTrDJn3/+mdtuu40GDRoQHR3Nxx87Wr+LFy/OIjJbvHhxPv/882z9jBgxgqioKGJjY2ndujW7du3KUv7LL79QuXJl+vTpA8CpU6do37499erVIzo6mgEDBnh1x48fT/ny5b0+x44d65VNmDCB2rVrU7t2bSZMmODl33LLLd7Ye/fuTXp6utdW5cqVsz2hFkxwNjchXIA333wTEeHwYUdZZvLkycTGxhIbG8t1113HunXrcvwMDMMwLkoK2kvckiXVvD/FNnz4cO3cubO2b99e09PTNSIiQrds2aKqqoMGDdKxY8eqquqrr77qCdYeOnRIy5YtqykpKVnaOnLkiJYtWzZbYElVRyQ2M/+9997Te++9N0v5n/70J+3cubM+8cQTquoIvi5atEhVHQHdG264QefNm6eqqh9//LFXz7//GjVq6JEjR/To0aNao0YNPXr0qKr+KgabkZGhd911l06ZMkVVVbt27eqJ3PoSTHA2J/FaVSdAZnx8vFatWtUT612+fLk3jnnz5ukf/vCHoJ9BqBLKTx2FEmanvBHKdsKeYsu3dEHvIInIfwt6DMEQkd4i8tA5bC9ORNr5vO8gIgNyuucc9t1CRObmXjN/2bt3L//617/o1asXAEeOHKFYsWLUqVMHcKQvZsyYATiBIE+cOIGqcvLkScLCwrLpi3322WfceuutWQJLZtKyZUsvv1mzZuzdu9crW7VqFQcPHswikFuyZElatnRikhYtWpRGjRpluScQ8+fPp02bNoSFhVG2bFnatGnDF184obQyxWDT0tI4c+ZMrmKwJUuW9ObnKzgbHh5Oo0aNgKzitZk8/fTTvPHGG1nav+666zxRXP+5+38GhmEYFysX9AJJ80mw9ly0o6pjVHXiuWjLJQ5HkiSz/Tmq+to5bL9AyRSrDZQy6du3L2+88QaFCjlf23LlypGamsq3334LOAuePXv2ANCnTx82b95MpUqVqF+/PiNHjvTuy2Tq1KnZgjwG4qOPPuLWW28FnKja/fr1Y9iwYUHrHz9+nH/+85+0bt3ay5sxYwaxsbF07NjRG+O+ffuoUqWKVyciIiLL4qVt27ZUqFCByy+/nI4dO3r577zzDrGxsfTo0YNjx455+StXriQ6Opr69eszZsyYbAtCf/HaOXPmULlyZRo0aJCnuUP2z8AwDONi5YJ20jbB2vwTrBWRW4C3gcPAap/8nARra+Bo4tUBngGaAbfi6OPdpqqpfn3kSaw2ISGBFStWkJqayokTJ1i7di1HjhxhyZIlPP/88/To0YPU1FSaNGlCcnIyCQkJLFmyhHLlyvHJJ5+wf/9+evXqxdixYylVqhTg7D6tXr2a4sWL5yhCuWDBAhYtWsTbb79NQkICs2bNom7dumzfvp3vv/+effv2Zbk/PT2dF198kXbt2rF79252795N2bJlmTBhAkWLFmXOnDncfvvtjBgxgm3btpGamurdv3PnzizjeeGFFzhz5gyvvPIKb731Fk2aNKF169Z06dIFEWHcuHHcf//99O/f3+s/mOCsv3htcnIy/fv3Z9iwYSQkJJCcnMzy5cspU+ZX5Zg1a9YwevRoRo0aFfQzCFUBTxMXzRtmp7xhdrpEKegzvt+TMMHafBGsdW2yx7WR4GFcjPsAABSOSURBVCwKMyOS5yRYuwy4DGiAs3i61S2bhbNIDfpZ5uaDNGDAAK1cubJWq1ZNK1asqCVKlNAHHnggS5358+frPffco6qq7dq106VLl3plLVu21JUrV3rv3377bX344Ydz7HPBggVar149PXjwoJd3//33a5UqVbRatWp61VVX6eWXX679+/f3yrt3765PPvlk0DbT0tL0iiuuUFXVTz75RB955BGv7JFHHtFPPvkk2z3jx4/3fJh8fSFyEpH1FZwNJF67fv16LV++vCeeW7hwYa1SpYoeOHBAVVXXrVunNWvW9Py7VPP2GYQKoewzEkqYnfJGKNsJ80HKvzVGQQ/gdw0+6wJpgU/+UuB697oV8Ll7PR5n98a3Xpy72PjYJ/9NHLmNtW7aBvTE2RnZifMofqZ8SBEcOZGPcERdi7r5Q4Bn3UXUbp+2awGr3esEn3FWBLblMNfcFkgPuNc1ga0+9SbiSKP8H85uUOacvgM+CtJXHLDU530HnwXSGqCmT9ked45DgIFuXiEcLbfMReNfgb45fZZnIzWyePFiz0E4c/GSnJysrVq10oULF6qqau/evfWll15SVdUff/xRK1Wq5Dkhq6o2bdrUc6oOxOrVq7VmzZr6ww8/BK3j73w9cOBAveuuuzQ9PT1Lvf3793vXM2fO1KZNm6qq46RdvXp1PXr0qB49elSrV6+uR44c0RMnTnj3pKam6r333qujR49WVdXPPvvMa2vEiBF63333qarqjh07PCftxMREDQ8P159++kkzMjK0S5cu+tRTTwWdh6pqtWrVPPvs2rVLa9WqpcuXLw9a3/czCEVC+QctlDA75Y1QtpMtkPIvXdBHbH7kh2DtB/6diEhjnN2eoSLypar+VUT+ALQGOgF9cBZlZzvu/BasTcdZRObudOMQLMR6MMFabwyqmiEiqe4fr+8YzjnDhg1j7ty5ZGRk8Nhjj9GqlWP6QYMG0a1bN+rXr4+q8vrrr3sCrImJiezZs4ebb745S1uDBw+mSZMmdOjQgeeee46TJ09yzz33AFC1alXmzJkTdBx79+7l1VdfpV69ep5TdJ8+fejVqxejRo1izpw5FClShLCwMMaPHw9AWFgYgwYN4tprr/X6DwsL4+DBg3To0IGUlBTS09Np1aqVJ777wQcfMGTIEESE6tWr88EHzld02bJlvPbaa1x22WUUKlTIE5xdtmwZkyZNon79+sTFxQHwt7/9jXbt2hGMv/71rxw5coTHH38cgCJFinh+XoZhGJcMBb1C+z0JE6zNF8Fa1ya7gVru+yk+NgwmWOvN1/ezCVQWKJlYbd4I5f/JhhJmp7xhdsoboWwnbAcp39LFtIOUV0ywNhfBWtcmjwD/EpHDOL5FMW7xEAIL1hqGYRjGRcMlI1YLJlgbyphYbd4IZdHMUMLslDfMTnkjlO1kYrX5hwUzMQzDMAzD8OOSOmJTE6wN1J8J1hqGYRiGH7aDFGLoJS5Y26NHDypUqEBMTIyXd/ToUdq0aUPt2rVp06aNFz06J1HVkSNHEhMTQ3R0NG+//XbAvmbPnk1sbCxxcXE0adKEZcuWAXkTtH3yyScpXbq09/7pp5/26tepU4crr7zSKytcuLBX1qFDh2zj8G8rJSWF++67j2uuuYamTZuSmJjozbdXr15eW4UKFWLt2rUADBw4kCpVqmRpB4KL7u7atYvGjRsTFxdHdHQ0Y8aM8e5p0aIFdevW9fo5dOhQrnM0DMO46ChoL3FLllR/fYptyZIlumrVqiwBEJ977jkdOnSoqqoOHTrUE6ENJqq6YcMGjY6O9gRcW7duHTCe0YkTJzQjI0NVncCIdevWzVYnkKDtN998ow8++KCWKlUqW31V1VGjRmn37t2998HqBWvr3Xff1UcffVRVVadMmZJFKDfzaZr169drjRo1vPwVK1bo/v37s/UVTHQ3JSVFk5OTPTtUq1ZN9+3bp6qqN998sxdkMhj+cww1Qvmpo1DC7JQ3QtlO2FNs+ZZsB8n4XaK/IjJHRDb65T0jIt+LyAYRWSciI0Tksry0d9NNNxEWFpYlb/bs2XTt6jws17VrV283J5io6ubNm2nWrJkn4HrzzTcza9asbH2VLl3aE2lNSkoKKAjrL2ibnp7Oc889xxtvvBF0DlOmTMmTxluwtnzn27FjRxYuXIjz72DwPpo1a0Z4eHi2PoKJ7hYtWpRixZyHGVNSUsjIyMh1vDn1bxiGcbFhCyQD/Y2ivyJyF04IBN+83jhxpJqpan3gWuAQjnZcUE6npgctO3jwoPfjHx4e7h35+OIrqhoTE8PSpUs5cuQIp06dYt68eZ5ArD+zZs2iXr16tG/fnnHjxmUr9xe0feedd+jQoUPAxQg4R1c7d+70AlYCJCcn06RJE5o1a5blqC5YW74itkWKFKFMmTIcOXIkS51p06ad9QLFX3h2z549xMbGUqVKFfr370+lSpW8su7duxMXF8fLL7+cbXEWaI6GYRgXG5eUk7YRGB/R33BgGnAFznfjMQ0gnuveUxpHkPYRHK22TAYCN6nqcQBVPQO8FqSNLGK1mWKQP/74I0lJSd77tLS0LEKR/u/9RVUBbr/9dpo3b06JEiWoVq0aP/74Y0CxybJlyzJmzBjWrVtHnz59GD58uFfmL2h7+PBhxo4d6wnXpqenZ2tzypQpNG/enK+++tVsU6dOpVy5cuzfv5/evXuTlJREsWLFgrZ18uRJVqxYQfny5QGyCMmePHmS9957D1Xl8OHD2foPNCbILrqbyahRozh8+DCDBg0iPDycsLAwnnjiCcqXL8+pU6d46aWXOHXqFG3bts1xjqGGiYvmDbNT3jA7XaIU9BmfpYJP/BqRvB+/6qkVBi7P4Z63gDvxidyNEzTz2G8Zg28kbX8R1jp16njaZPv371ffuoFEVf154YUX9N133w1ankn16tWz6LX5C9rOnTtXK1as6Am8iojWqlUrSxtxcXE5aph17dpVp0+fnmNb8fHx+t///ldVHS22q666yvOVWrx4sfbt21dfffXVgO0H8ncKJLrrT7du3XT69OnZ8v315vIyx1AglH1GQgmzU94IZTthPkj5luyIzfDlG6C7iAwB6qvqiUCVRCQOR7LE37FH8NFwE5G2IrJWRBJF5Dcd4wF06NCBCRMmADBhwgRuv/12AHbv3s1dd93FpEmTqFOnTpZ7Mo/hdu/ezcyZMwMeR23bti1zscfq1as5c+YMV111lVfu72fTvn17fvzxRxITE0lMTKRkyZJs27bNK9+yZQvHjh2jefPmXt6xY8dISXGk8Q4fPszy5cuJiorKsS3f+X722We0atXK84/KyMhg+vTpdOrUKU+2W7NmDY8++ihz5syhQoUKXv7evXs5ffq0N8bly5dTt25d0tLSOHz4MACpqanMnTs3yxOFgeZoGIZxMWJHbIaHqi4VkZuA9sAkERmmqhMDVG0ONBaRRJzvUAURSVDVFiKSJCI1VHWnOuEC5ovIXKBoXsbQuXNn7zgrIiKCv/zlLwwYMIB7772Xjz76iKpVqzJ9+nQgZ1HVu+++myNHjnDZZZfx7rvves7cmY+z9+7dmxkzZjBx4kQuu+wySpQowbRp07yFSDBB25yYMmUKnTp1yuLsvXnzZh599FEKFSpERkYGAwYMICoqKsd2evbsSZcuXbjmmmsICwtj6tSpXtn69euJiIigZs2aWe55/vnn+eSTTzh16hQRERH06tWLIUOGBBXd3bx5M/369UNEUFWeffZZ6tevT1JSEm3btiU1NZX09HT++Mc/8vDDD+c4R8MwjIuRS0pqxAiMjw9SNWCfqqaJSF+guqr2zeXe6jjyLTHu+8eBDkAnVT0uzi/pAuAVVU0I1o5JjeSNUJY8CCXMTnnD7JQ3QtlOJjWSf9gOkuFLC+A5EUnFeTrtod/QxvtASWCliKS47SwH1pyrQRqGYRhGfmMLJANVLe2+TgAmnOW9iUCMz3sF3nSTYRiGYVyQmJO2YRiGYRiGH7aDZOSIiKwEivlld1HVDQUxHsMwDMM4H9gCycgRVW1a0GMwDMMwjPONHbEZhmEYhmH4YQskwzAMwzAMP2yBZBiGYRiG4YcFijRCAhE5AVikyNwpBxwu6EFcAJid8obZKW+Esp2qqWr5gh7ExYg5aRuhwhaLBps7IvKt2Sl3zE55w+yUN8xOlyZ2xGYYhmEYhuGHLZAMwzAMwzD8sAWSESr8vaAHcIFgdsobZqe8YXbKG2anSxBz0jYMwzAMw/DDdpAMwzAMwzD8sAWSYRiGYRiGH7ZAMgoUEblFRLaIyDYRGVDQ4zlfiEiiiGwQkbUi8q2bFyYiC0Rkq/ta1s0XERnl2mi9iDTyaaerW3+riHT1yW/str/NvVfO/yzPHhEZJyKHRGSjT16+2yVYH6FKEDsNEZF97ndqrYi08yl7wZ3zFhFp65Mf8O9PRGqIyErXHtNEpKibX8x9v80tr35+ZvzbEJEqIrJYRDaLyCYRecrNt++UkTuqaslSgSSgMLAdqAkUBdYBUQU9rvM090SgnF/eG8AA93oA8Lp73Q74NyBAM2Clmx8G7HBfy7rXZd2y/wHN3Xv+Ddxa0HPOo11uAhoBG8+nXYL1EaopiJ2GAM8GqBvl/m0VA2q4f3OFc/r7Az4FOrnXY4DH3OvHgTHudSdgWkHbIhc7hQON3OvLgR9ce9h3ylKuyXaQjILkD8A2Vd2hqmeAqcDtBTymguR2YIJ7PQG4wyd/ojp8DVwpIuFAW2CBqh5V1WPAAuAWt+wKVV2hzr/OE33aCmlUdSlw1C/7fNglWB8hSRA7BeN2YKqqpqjqTmAbzt9ewL8/dwekFfCZe7+/zTPt9BnQOpR3J1X1gKqudq9PAJuByth3ysgDtkAyCpLKwB6f93vdvEsBBb4UkVUi8oibV1FVD4DzDztQwc0PZqec8vcGyL9QOR92CdbHhUYf92honM+Rztna6SrguKqm+eVnacst/9mtH/K4x4ENgZXYd8rIA7ZAMgqSQP/zvFTiTlyvqo2AW4EnROSmHOoGs9PZ5l9smF2y8j5QC4gDDgDD3fxzaacL0oYiUhqYAfRV1V9yqhog71L+Tl3S2ALJKEj2AlV83kcA+wtoLOcVVd3vvh4CZuEcdxx0t+xxXw+51YPZKaf8iAD5Fyrnwy7B+rhgUNWDqpquqhnAhzjfKTh7Ox3GOVoq4pefpS23vAx5P+orEETkMpzF0WRVnelm23fKyBVbIBkFyTdAbfeJmaI4Tp9zCnhM+Y6IlBKRyzOvgXhgI87cM5+O6QrMdq/nAA+5T9g0A352t+znA/EiUtY9TokH5rtlJ0Skmesf8pBPWxci58Muwfq4YMj8MXa5E+c7Bc7cOrlPoNUAauM4Fgf8+3N9aRYDHd37/W2eaaeOwCK3fkjifs4fAZtVdYRPkX2njNwpaC9xS5d2wnlq5Aecp2kGFvR4ztOca+I8MbQO2JQ5bxxfjoXAVvc1zM0X4F3XRhuAJj5t9cBxut0GdPfJb4LzA7kdeAc3an6oJ2AKzvFQKs7/znueD7sE6yNUUxA7TXLtsB7nxzncp/5Ad85b8HmiMdjfn/sd/Z9rv+lAMTe/uPt+m1tes6BtkYudbsA58loPrHVTO/tOWcpLMqkRwzAMwzAMP+yIzTAMwzAMww9bIBmGYRiGYfhhCyTDMAzDMAw/bIFkGIZhGIbhhy2QDMMwDMMw/LAFkmEY+YqIpMuvCvNrf4sCvIhcKSKPn/vRee138FWzPx+IyB0iEnU++zQMI+/YY/6GYeQrInJSVUv/zjaqA3NVNeYs7yusqum/p+/8wI1CPRZnTp/lVt8wjPOP7SAZhnHeEZHCIjJMRL5xxVUfdfNLi8hCEVktIhtE5Hb3lteAWu4O1DARaSEic33ae0dEurnXiSIyWESWAfeISC0R+cIVBv5KROoFGE83EXnHvR4vIu+LyGIR2SEiN7vir5tFZLzPPSdFZLg71oUiUt7NjxORr915zcoUjRWRBBH5m4gsAfoDHYBh7pxqicjDrj3WicgMESnpM55RIvJfdzwdfcbwvGundSLympuX63wNw8idIrlXMQzD+F2UEJG17vVOVb0TJ/Lzz6p6rYgUA5aLyJc4iul3quovIlIO+FpE5gADgBhVjQMQkRa59Jmsqje4dRcCvVV1q4g0Bd4DWuVyf1m3Tgfgn8D1QC/gGxGJU9W1QClgtar2E5HBwEtAH2Ai8KSqLhGRv7r5fd12r1TVm91x1cZnB0lEjqvqh+71K66NRrv3heNEha6HEyX7MxG5FbgDaKqqp0QkzK37998wX8Mw/LAFkmEY+c3pzIWND/FArM9uSBkcjbC9wN9E5CYgA6gMVPwNfU4DT8X9OmC6I5UFQLE83P9PVVUR2QAcVNUNbnubgOo4khUZmf0A/wBmikgZnEXQEjd/Ao40R5ZxBSHGXRhdCZTG0f/K5HN1RGi/E5FMe/wR+FhVTwGo6tHfMV/DMPywBZJhGAWB4OyyzM+S6RyTlQcaq2qqiCTi6H/5k0ZWFwH/OknuayHgeIAFWm6kuK8ZPteZ74P9u5kXh86kHMrGA3eo6jrXDi0CjAcc22W++vf5W+drGIYf5oNkGEZBMB94TEQuAxCROiJSCmcn6ZC7OGoJVHPrnwAu97l/FxAljkJ9GaB1oE5U9Rdgp4jc4/YjItLgHM2hEL8q3t8PLFPVn4FjInKjm98FWBLoZrLP6XLggGuTB/LQ/5dADx9fpbB8nq9hXFLYAskwjIJgLPAdsFpENgIf4OzMTAaaiMi3OIuE7wFU9QiOn9JGERmmqnuAT3FU2icDa3Lo6wGgp4isAzYBt+dQ92xIAqJFZBWOj89f3fyuOM7X64E4n3x/pgLPicgaEakFDAJWAgtw550TqvoFjj/St66P17NuUX7N1zAuKewxf8MwjN+AnIPwBYZhhC62g2QYhmEYhuGH7SAZhmEYhmH4YTtIhmEYhmEYftgCyTAMwzAMww9bIBmGYRiGYfhhCyTDMAzDMAw/bIFkGIZhGIbhx/8DLVKl4Y84cS8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lightgbm.plot_importance(lgb, importance_type=\"gain\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = neural_network.MLPClassifier(hidden_layer_sizes=(25, 10), verbose=1, random_state=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = preprocessing.StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f_tr_filled = df_f_tr.copy()\n",
    "df_f_te_filled = df_f_te.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is_4G\n",
      "visit_number\n",
      "impression_time_hour\n",
      "impression_time_dom\n",
      "user_click_ewm\n",
      "user_click_ewm2\n",
      "n_ad_seen\n",
      "prev_add_seen_days_ago\n",
      "prev_click_sum\n",
      "app_code_count_n\n",
      "app_click_ewm\n",
      "prev_app_click\n",
      "prev_app_click_sum\n",
      "prev_user_app_click_mean\n",
      "user_count_leak\n",
      "app_count_leak\n"
     ]
    }
   ],
   "source": [
    "for col in predictors:\n",
    "    mean_ = df_f_tr[col].mean()\n",
    "    print(col)\n",
    "    df_f_tr_filled[col] = df_f_tr_filled[col].fillna(mean_)\n",
    "    df_f_te_filled[col] = df_f_te_filled[col].fillna(mean_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/data.py:625: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "tr_sc = scaler.fit_transform(df_f_tr_filled[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "te_sc = scaler.transform(df_f_te_filled[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 0.22083339\n",
      "Iteration 2, loss = 0.17349259\n",
      "Iteration 3, loss = 0.17240066\n",
      "Iteration 4, loss = 0.17179769\n",
      "Iteration 5, loss = 0.17139478\n",
      "Iteration 6, loss = 0.17107568\n",
      "Iteration 7, loss = 0.17101772\n",
      "Iteration 8, loss = 0.17089016\n",
      "Iteration 9, loss = 0.17071538\n",
      "Iteration 10, loss = 0.17061604\n",
      "Iteration 11, loss = 0.17051365\n",
      "Iteration 12, loss = 0.17045071\n",
      "Iteration 13, loss = 0.17034039\n",
      "Iteration 14, loss = 0.17034513\n",
      "Iteration 15, loss = 0.17021090\n",
      "Iteration 16, loss = 0.17018129\n",
      "Iteration 17, loss = 0.17016966\n",
      "Iteration 18, loss = 0.17005310\n",
      "Iteration 19, loss = 0.17001507\n",
      "Iteration 20, loss = 0.17005453\n",
      "Iteration 21, loss = 0.17002015\n",
      "Iteration 22, loss = 0.16982289\n",
      "Iteration 23, loss = 0.16988119\n",
      "Iteration 24, loss = 0.16989444\n",
      "Iteration 25, loss = 0.16983195\n",
      "Iteration 26, loss = 0.16976187\n",
      "Iteration 27, loss = 0.16974876\n",
      "Iteration 28, loss = 0.16975847\n",
      "Iteration 29, loss = 0.16969730\n",
      "Iteration 30, loss = 0.16951427\n",
      "Iteration 31, loss = 0.16955108\n",
      "Iteration 32, loss = 0.16952609\n",
      "Iteration 33, loss = 0.16947675\n",
      "Iteration 34, loss = 0.16947153\n",
      "Iteration 35, loss = 0.16939264\n",
      "Iteration 36, loss = 0.16930321\n",
      "Iteration 37, loss = 0.16943566\n",
      "Iteration 38, loss = 0.16926601\n",
      "Iteration 39, loss = 0.16923799\n",
      "Iteration 40, loss = 0.16925908\n",
      "Iteration 41, loss = 0.16919039\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(25, 10), learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
       "       random_state=1000, shuffle=True, solver='adam', tol=0.0001,\n",
       "       validation_fraction=0.1, verbose=1, warm_start=False)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.fit(tr_sc, df_f_tr[\"is_click\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = nn.predict_proba(te_sc)[:,1]\n",
    "preds_tr = nn.predict_proba(tr_sc)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7315617760449162, 0.7297852262448479)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.roc_auc_score(df_f_tr[\"is_click\"], preds_tr), metrics.roc_auc_score(df_f_te[\"is_click\"], preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f_filled = df_f.copy()\n",
    "df_ft_filled = df_ft.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is_4G\n",
      "visit_number\n",
      "impression_time_hour\n",
      "impression_time_dom\n",
      "user_click_ewm\n",
      "user_click_ewm2\n",
      "n_ad_seen\n",
      "prev_add_seen_days_ago\n",
      "prev_click_sum\n",
      "app_code_count_n\n",
      "app_click_ewm\n",
      "prev_app_click\n",
      "prev_app_click_sum\n",
      "prev_user_app_click_mean\n",
      "user_count_leak\n",
      "app_count_leak\n"
     ]
    }
   ],
   "source": [
    "for col in predictors:\n",
    "    mean_ = df_f[col].mean()\n",
    "    print(col)\n",
    "    df_f_filled[col] = df_f_filled[col].fillna(mean_)\n",
    "    df_ft_filled[col] = df_ft_filled[col].fillna(mean_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/data.py:625: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "tr_f_sc = scaler.fit_transform(df_f_filled[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "te_f_sc = scaler.transform(df_ft_filled[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "Iteration 1, loss = 0.20521954\n",
      "Iteration 2, loss = 0.17247245\n",
      "Iteration 3, loss = 0.17164836\n",
      "Iteration 4, loss = 0.17126577\n",
      "Iteration 5, loss = 0.17099261\n",
      "Iteration 6, loss = 0.17089058\n",
      "Iteration 7, loss = 0.17080486\n",
      "Iteration 8, loss = 0.17072739\n",
      "Iteration 9, loss = 0.17060419\n",
      "Iteration 10, loss = 0.17052461\n",
      "Iteration 11, loss = 0.17043661\n",
      "Iteration 12, loss = 0.17039701\n",
      "Iteration 13, loss = 0.17031689\n",
      "Iteration 14, loss = 0.17027241\n",
      "Iteration 15, loss = 0.17019154\n",
      "Iteration 16, loss = 0.17008047\n",
      "Iteration 17, loss = 0.17006118\n",
      "Iteration 18, loss = 0.17005557\n",
      "Iteration 19, loss = 0.16995113\n",
      "Iteration 20, loss = 0.16990861\n",
      "Iteration 21, loss = 0.16993281\n",
      "Iteration 22, loss = 0.16987339\n",
      "Iteration 23, loss = 0.16988444\n",
      "Iteration 24, loss = 0.16981340\n",
      "Iteration 25, loss = 0.16977543\n",
      "Iteration 26, loss = 0.16970418\n",
      "Iteration 27, loss = 0.16975709\n",
      "Iteration 28, loss = 0.16969352\n",
      "Iteration 29, loss = 0.16966912\n",
      "Iteration 30, loss = 0.16960936\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1001\n",
      "Iteration 1, loss = 0.19387249\n",
      "Iteration 2, loss = 0.17256261\n",
      "Iteration 3, loss = 0.17169706\n",
      "Iteration 4, loss = 0.17137244\n",
      "Iteration 5, loss = 0.17104495\n",
      "Iteration 6, loss = 0.17081937\n",
      "Iteration 7, loss = 0.17074260\n",
      "Iteration 8, loss = 0.17061079\n",
      "Iteration 9, loss = 0.17053760\n",
      "Iteration 10, loss = 0.17043617\n",
      "Iteration 11, loss = 0.17033866\n",
      "Iteration 12, loss = 0.17029581\n",
      "Iteration 13, loss = 0.17023545\n",
      "Iteration 14, loss = 0.17016590\n",
      "Iteration 15, loss = 0.17009900\n",
      "Iteration 16, loss = 0.17001312\n",
      "Iteration 17, loss = 0.17001988\n",
      "Iteration 18, loss = 0.16990027\n",
      "Iteration 19, loss = 0.16983335\n",
      "Iteration 20, loss = 0.16980870\n",
      "Iteration 21, loss = 0.16977823\n",
      "Iteration 22, loss = 0.16973141\n",
      "Iteration 23, loss = 0.16973051\n",
      "Iteration 24, loss = 0.16965728\n",
      "Iteration 25, loss = 0.16961441\n",
      "Iteration 26, loss = 0.16956413\n",
      "Iteration 27, loss = 0.16956456\n",
      "Iteration 28, loss = 0.16954509\n",
      "Iteration 29, loss = 0.16950061\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1002\n",
      "Iteration 1, loss = 0.19659879\n",
      "Iteration 2, loss = 0.17290819\n",
      "Iteration 3, loss = 0.17193826\n",
      "Iteration 4, loss = 0.17151589\n",
      "Iteration 5, loss = 0.17122090\n",
      "Iteration 6, loss = 0.17105296\n",
      "Iteration 7, loss = 0.17090012\n",
      "Iteration 8, loss = 0.17077096\n",
      "Iteration 9, loss = 0.17071251\n",
      "Iteration 10, loss = 0.17066283\n",
      "Iteration 11, loss = 0.17054076\n",
      "Iteration 12, loss = 0.17053247\n",
      "Iteration 13, loss = 0.17047493\n",
      "Iteration 14, loss = 0.17040718\n",
      "Iteration 15, loss = 0.17046779\n",
      "Iteration 16, loss = 0.17028124\n",
      "Iteration 17, loss = 0.17024550\n",
      "Iteration 18, loss = 0.17021781\n",
      "Iteration 19, loss = 0.17024443\n",
      "Iteration 20, loss = 0.17011080\n",
      "Iteration 21, loss = 0.17007302\n",
      "Iteration 22, loss = 0.17009736\n",
      "Iteration 23, loss = 0.17000897\n",
      "Iteration 24, loss = 0.16996661\n",
      "Iteration 25, loss = 0.16995023\n",
      "Iteration 26, loss = 0.16989383\n",
      "Iteration 27, loss = 0.16988668\n",
      "Iteration 28, loss = 0.16986386\n",
      "Iteration 29, loss = 0.16984931\n",
      "Iteration 30, loss = 0.16979788\n",
      "Iteration 31, loss = 0.16977700\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1003\n",
      "Iteration 1, loss = 0.18631064\n",
      "Iteration 2, loss = 0.17179607\n",
      "Iteration 3, loss = 0.17126397\n",
      "Iteration 4, loss = 0.17097238\n",
      "Iteration 5, loss = 0.17079285\n",
      "Iteration 6, loss = 0.17064780\n",
      "Iteration 7, loss = 0.17050701\n",
      "Iteration 8, loss = 0.17045377\n",
      "Iteration 9, loss = 0.17035722\n",
      "Iteration 10, loss = 0.17034636\n",
      "Iteration 11, loss = 0.17024947\n",
      "Iteration 12, loss = 0.17021188\n",
      "Iteration 13, loss = 0.17016969\n",
      "Iteration 14, loss = 0.17009697\n",
      "Iteration 15, loss = 0.16999721\n",
      "Iteration 16, loss = 0.16998438\n",
      "Iteration 17, loss = 0.16995812\n",
      "Iteration 18, loss = 0.16994440\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1004\n",
      "Iteration 1, loss = 0.22833254\n",
      "Iteration 2, loss = 0.17327646\n",
      "Iteration 3, loss = 0.17195668\n",
      "Iteration 4, loss = 0.17146095\n",
      "Iteration 5, loss = 0.17129131\n",
      "Iteration 6, loss = 0.17103610\n",
      "Iteration 7, loss = 0.17087084\n",
      "Iteration 8, loss = 0.17080629\n",
      "Iteration 9, loss = 0.17061922\n",
      "Iteration 10, loss = 0.17066881\n",
      "Iteration 11, loss = 0.17051985\n",
      "Iteration 12, loss = 0.17043659\n",
      "Iteration 13, loss = 0.17042657\n",
      "Iteration 14, loss = 0.17033166\n",
      "Iteration 15, loss = 0.17031357\n",
      "Iteration 16, loss = 0.17026780\n",
      "Iteration 17, loss = 0.17021974\n",
      "Iteration 18, loss = 0.17019002\n",
      "Iteration 19, loss = 0.17010579\n",
      "Iteration 20, loss = 0.17009976\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1005\n",
      "Iteration 1, loss = 0.18160988\n",
      "Iteration 2, loss = 0.17198420\n",
      "Iteration 3, loss = 0.17134385\n",
      "Iteration 4, loss = 0.17107738\n",
      "Iteration 5, loss = 0.17086939\n",
      "Iteration 6, loss = 0.17070132\n",
      "Iteration 7, loss = 0.17069318\n",
      "Iteration 8, loss = 0.17061179\n",
      "Iteration 9, loss = 0.17053400\n",
      "Iteration 10, loss = 0.17040034\n",
      "Iteration 11, loss = 0.17032065\n",
      "Iteration 12, loss = 0.17033048\n",
      "Iteration 13, loss = 0.17023871\n",
      "Iteration 14, loss = 0.17020479\n",
      "Iteration 15, loss = 0.17017257\n",
      "Iteration 16, loss = 0.17006847\n",
      "Iteration 17, loss = 0.17011701\n",
      "Iteration 18, loss = 0.17006337\n",
      "Iteration 19, loss = 0.16998875\n",
      "Iteration 20, loss = 0.16999358\n",
      "Iteration 21, loss = 0.16989843\n",
      "Iteration 22, loss = 0.16986959\n",
      "Iteration 23, loss = 0.16983460\n",
      "Iteration 24, loss = 0.16982666\n",
      "Iteration 25, loss = 0.16986450\n",
      "Iteration 26, loss = 0.16977824\n",
      "Iteration 27, loss = 0.16975753\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1006\n",
      "Iteration 1, loss = 0.18240883\n",
      "Iteration 2, loss = 0.17180890\n",
      "Iteration 3, loss = 0.17133568\n",
      "Iteration 4, loss = 0.17112476\n",
      "Iteration 5, loss = 0.17094302\n",
      "Iteration 6, loss = 0.17079092\n",
      "Iteration 7, loss = 0.17069443\n",
      "Iteration 8, loss = 0.17055405\n",
      "Iteration 9, loss = 0.17050583\n",
      "Iteration 10, loss = 0.17042274\n",
      "Iteration 11, loss = 0.17031329\n",
      "Iteration 12, loss = 0.17021540\n",
      "Iteration 13, loss = 0.17015371\n",
      "Iteration 14, loss = 0.17009835\n",
      "Iteration 15, loss = 0.17004526\n",
      "Iteration 16, loss = 0.16993214\n",
      "Iteration 17, loss = 0.16995988\n",
      "Iteration 18, loss = 0.16986023\n",
      "Iteration 19, loss = 0.16976620\n",
      "Iteration 20, loss = 0.16978799\n",
      "Iteration 21, loss = 0.16975331\n",
      "Iteration 22, loss = 0.16966745\n",
      "Iteration 23, loss = 0.16965681\n",
      "Iteration 24, loss = 0.16960158\n",
      "Iteration 25, loss = 0.16964504\n",
      "Iteration 26, loss = 0.16958342\n",
      "Iteration 27, loss = 0.16954760\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1007\n",
      "Iteration 1, loss = 0.18067187\n",
      "Iteration 2, loss = 0.17232247\n",
      "Iteration 3, loss = 0.17155568\n",
      "Iteration 4, loss = 0.17128968\n",
      "Iteration 5, loss = 0.17107418\n",
      "Iteration 6, loss = 0.17089463\n",
      "Iteration 7, loss = 0.17070138\n",
      "Iteration 8, loss = 0.17059952\n",
      "Iteration 9, loss = 0.17056309\n",
      "Iteration 10, loss = 0.17042833\n",
      "Iteration 11, loss = 0.17028491\n",
      "Iteration 12, loss = 0.17035662\n",
      "Iteration 13, loss = 0.17016443\n",
      "Iteration 14, loss = 0.17011487\n",
      "Iteration 15, loss = 0.17008226\n",
      "Iteration 16, loss = 0.16999022\n",
      "Iteration 17, loss = 0.16999173\n",
      "Iteration 18, loss = 0.16991730\n",
      "Iteration 19, loss = 0.16987995\n",
      "Iteration 20, loss = 0.16984154\n",
      "Iteration 21, loss = 0.16973987\n",
      "Iteration 22, loss = 0.16979108\n",
      "Iteration 23, loss = 0.16966872\n",
      "Iteration 24, loss = 0.16970397\n",
      "Iteration 25, loss = 0.16963845\n",
      "Iteration 26, loss = 0.16960481\n",
      "Iteration 27, loss = 0.16956716\n",
      "Iteration 28, loss = 0.16951989\n",
      "Iteration 29, loss = 0.16953319\n",
      "Iteration 30, loss = 0.16952567\n",
      "Iteration 31, loss = 0.16948211\n",
      "Iteration 32, loss = 0.16942682\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1008\n",
      "Iteration 1, loss = 0.19238436\n",
      "Iteration 2, loss = 0.17223231\n",
      "Iteration 3, loss = 0.17155579\n",
      "Iteration 4, loss = 0.17115543\n",
      "Iteration 5, loss = 0.17092651\n",
      "Iteration 6, loss = 0.17078232\n",
      "Iteration 7, loss = 0.17067683\n",
      "Iteration 8, loss = 0.17051798\n",
      "Iteration 9, loss = 0.17032459\n",
      "Iteration 10, loss = 0.17033269\n",
      "Iteration 11, loss = 0.17021945\n",
      "Iteration 12, loss = 0.17018495\n",
      "Iteration 13, loss = 0.17006334\n",
      "Iteration 14, loss = 0.17005024\n",
      "Iteration 15, loss = 0.16993768\n",
      "Iteration 16, loss = 0.16986550\n",
      "Iteration 17, loss = 0.16981902\n",
      "Iteration 18, loss = 0.16973949\n",
      "Iteration 19, loss = 0.16970101\n",
      "Iteration 20, loss = 0.16965720\n",
      "Iteration 21, loss = 0.16961805\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 22, loss = 0.16962330\n",
      "Iteration 23, loss = 0.16954494\n",
      "Iteration 24, loss = 0.16953435\n",
      "Iteration 25, loss = 0.16951496\n",
      "Iteration 26, loss = 0.16943978\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1009\n",
      "Iteration 1, loss = 0.19651662\n",
      "Iteration 2, loss = 0.17204132\n",
      "Iteration 3, loss = 0.17137885\n",
      "Iteration 4, loss = 0.17110277\n",
      "Iteration 5, loss = 0.17094105\n",
      "Iteration 6, loss = 0.17081154\n",
      "Iteration 7, loss = 0.17069066\n",
      "Iteration 8, loss = 0.17061285\n",
      "Iteration 9, loss = 0.17057615\n",
      "Iteration 10, loss = 0.17049845\n",
      "Iteration 11, loss = 0.17041165\n",
      "Iteration 12, loss = 0.17039059\n",
      "Iteration 13, loss = 0.17036522\n",
      "Iteration 14, loss = 0.17026730\n",
      "Iteration 15, loss = 0.17029466\n",
      "Iteration 16, loss = 0.17023509\n",
      "Iteration 17, loss = 0.17017364\n",
      "Iteration 18, loss = 0.17013081\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1010\n",
      "Iteration 1, loss = 0.20221411\n",
      "Iteration 2, loss = 0.17306329\n",
      "Iteration 3, loss = 0.17189286\n",
      "Iteration 4, loss = 0.17144405\n",
      "Iteration 5, loss = 0.17122894\n",
      "Iteration 6, loss = 0.17108199\n",
      "Iteration 7, loss = 0.17083381\n",
      "Iteration 8, loss = 0.17078067\n",
      "Iteration 9, loss = 0.17066107\n",
      "Iteration 10, loss = 0.17060009\n",
      "Iteration 11, loss = 0.17054511\n",
      "Iteration 12, loss = 0.17044599\n",
      "Iteration 13, loss = 0.17036664\n",
      "Iteration 14, loss = 0.17039533\n",
      "Iteration 15, loss = 0.17036934\n",
      "Iteration 16, loss = 0.17024998\n",
      "Iteration 17, loss = 0.17027924\n",
      "Iteration 18, loss = 0.17018096\n",
      "Iteration 19, loss = 0.17013108\n",
      "Iteration 20, loss = 0.17007560\n",
      "Iteration 21, loss = 0.16998815\n",
      "Iteration 22, loss = 0.17003487\n",
      "Iteration 23, loss = 0.16997891\n",
      "Iteration 24, loss = 0.16993979\n",
      "Iteration 25, loss = 0.16993920\n",
      "Iteration 26, loss = 0.16988255\n",
      "Iteration 27, loss = 0.16988230\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1011\n",
      "Iteration 1, loss = 0.18171916\n",
      "Iteration 2, loss = 0.17188723\n",
      "Iteration 3, loss = 0.17137044\n",
      "Iteration 4, loss = 0.17103798\n",
      "Iteration 5, loss = 0.17087451\n",
      "Iteration 6, loss = 0.17076387\n",
      "Iteration 7, loss = 0.17066166\n",
      "Iteration 8, loss = 0.17049903\n",
      "Iteration 9, loss = 0.17047281\n",
      "Iteration 10, loss = 0.17039124\n",
      "Iteration 11, loss = 0.17029250\n",
      "Iteration 12, loss = 0.17010704\n",
      "Iteration 13, loss = 0.17007539\n",
      "Iteration 14, loss = 0.17003265\n",
      "Iteration 15, loss = 0.16999065\n",
      "Iteration 16, loss = 0.16989820\n",
      "Iteration 17, loss = 0.16984216\n",
      "Iteration 18, loss = 0.16977439\n",
      "Iteration 19, loss = 0.16977193\n",
      "Iteration 20, loss = 0.16965912\n",
      "Iteration 21, loss = 0.16969487\n",
      "Iteration 22, loss = 0.16966367\n",
      "Iteration 23, loss = 0.16962016\n",
      "Iteration 24, loss = 0.16958003\n",
      "Iteration 25, loss = 0.16952313\n",
      "Iteration 26, loss = 0.16953897\n",
      "Iteration 27, loss = 0.16949300\n",
      "Iteration 28, loss = 0.16943322\n",
      "Iteration 29, loss = 0.16944188\n",
      "Iteration 30, loss = 0.16943579\n",
      "Iteration 31, loss = 0.16939963\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1012\n",
      "Iteration 1, loss = 0.23257547\n",
      "Iteration 2, loss = 0.17339323\n",
      "Iteration 3, loss = 0.17209324\n",
      "Iteration 4, loss = 0.17154484\n",
      "Iteration 5, loss = 0.17122404\n",
      "Iteration 6, loss = 0.17101859\n",
      "Iteration 7, loss = 0.17085904\n",
      "Iteration 8, loss = 0.17074656\n",
      "Iteration 9, loss = 0.17063141\n",
      "Iteration 10, loss = 0.17058467\n",
      "Iteration 11, loss = 0.17049375\n",
      "Iteration 12, loss = 0.17044659\n",
      "Iteration 13, loss = 0.17037910\n",
      "Iteration 14, loss = 0.17035360\n",
      "Iteration 15, loss = 0.17030174\n",
      "Iteration 16, loss = 0.17032344\n",
      "Iteration 17, loss = 0.17020850\n",
      "Iteration 18, loss = 0.17018172\n",
      "Iteration 19, loss = 0.17015213\n",
      "Iteration 20, loss = 0.17010166\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1013\n",
      "Iteration 1, loss = 0.19804920\n",
      "Iteration 2, loss = 0.17265594\n",
      "Iteration 3, loss = 0.17165269\n",
      "Iteration 4, loss = 0.17127873\n",
      "Iteration 5, loss = 0.17105645\n",
      "Iteration 6, loss = 0.17085528\n",
      "Iteration 7, loss = 0.17068312\n",
      "Iteration 8, loss = 0.17059925\n",
      "Iteration 9, loss = 0.17050396\n",
      "Iteration 10, loss = 0.17046230\n",
      "Iteration 11, loss = 0.17041953\n",
      "Iteration 12, loss = 0.17032995\n",
      "Iteration 13, loss = 0.17025601\n",
      "Iteration 14, loss = 0.17026254\n",
      "Iteration 15, loss = 0.17015067\n",
      "Iteration 16, loss = 0.17005976\n",
      "Iteration 17, loss = 0.17010030\n",
      "Iteration 18, loss = 0.16996783\n",
      "Iteration 19, loss = 0.16998875\n",
      "Iteration 20, loss = 0.16988211\n",
      "Iteration 21, loss = 0.16999205\n",
      "Iteration 22, loss = 0.16989403\n",
      "Iteration 23, loss = 0.16979736\n",
      "Iteration 24, loss = 0.16985859\n",
      "Iteration 25, loss = 0.16976052\n",
      "Iteration 26, loss = 0.16975820\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1014\n",
      "Iteration 1, loss = 0.19309010\n",
      "Iteration 2, loss = 0.17212911\n",
      "Iteration 3, loss = 0.17145807\n",
      "Iteration 4, loss = 0.17117341\n",
      "Iteration 5, loss = 0.17104206\n",
      "Iteration 6, loss = 0.17084621\n",
      "Iteration 7, loss = 0.17071986\n",
      "Iteration 8, loss = 0.17061778\n",
      "Iteration 9, loss = 0.17061348\n",
      "Iteration 10, loss = 0.17048733\n",
      "Iteration 11, loss = 0.17039256\n",
      "Iteration 12, loss = 0.17029412\n",
      "Iteration 13, loss = 0.17030195\n",
      "Iteration 14, loss = 0.17022855\n",
      "Iteration 15, loss = 0.17014597\n",
      "Iteration 16, loss = 0.17012050\n",
      "Iteration 17, loss = 0.17004261\n",
      "Iteration 18, loss = 0.16998058\n",
      "Iteration 19, loss = 0.16996945\n",
      "Iteration 20, loss = 0.16990258\n",
      "Iteration 21, loss = 0.16985708\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1015\n",
      "Iteration 1, loss = 0.20057178\n",
      "Iteration 2, loss = 0.17267124\n",
      "Iteration 3, loss = 0.17176733\n",
      "Iteration 4, loss = 0.17147334\n",
      "Iteration 5, loss = 0.17120321\n",
      "Iteration 6, loss = 0.17101561\n",
      "Iteration 7, loss = 0.17097113\n",
      "Iteration 8, loss = 0.17079364\n",
      "Iteration 9, loss = 0.17067295\n",
      "Iteration 10, loss = 0.17060963\n",
      "Iteration 11, loss = 0.17059343\n",
      "Iteration 12, loss = 0.17048035\n",
      "Iteration 13, loss = 0.17047120\n",
      "Iteration 14, loss = 0.17044846\n",
      "Iteration 15, loss = 0.17038103\n",
      "Iteration 16, loss = 0.17030915\n",
      "Iteration 17, loss = 0.17028125\n",
      "Iteration 18, loss = 0.17020523\n",
      "Iteration 19, loss = 0.17023178\n",
      "Iteration 20, loss = 0.17016223\n",
      "Iteration 21, loss = 0.17022306\n",
      "Iteration 22, loss = 0.17013636\n",
      "Iteration 23, loss = 0.17011149\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1016\n",
      "Iteration 1, loss = 0.20797564\n",
      "Iteration 2, loss = 0.17232817\n",
      "Iteration 3, loss = 0.17167791\n",
      "Iteration 4, loss = 0.17128331\n",
      "Iteration 5, loss = 0.17106509\n",
      "Iteration 6, loss = 0.17086160\n",
      "Iteration 7, loss = 0.17069681\n",
      "Iteration 8, loss = 0.17068817\n",
      "Iteration 9, loss = 0.17051128\n",
      "Iteration 10, loss = 0.17041900\n",
      "Iteration 11, loss = 0.17026135\n",
      "Iteration 12, loss = 0.17024213\n",
      "Iteration 13, loss = 0.17021656\n",
      "Iteration 14, loss = 0.17014461\n",
      "Iteration 15, loss = 0.17004056\n",
      "Iteration 16, loss = 0.17004587\n",
      "Iteration 17, loss = 0.16998920\n",
      "Iteration 18, loss = 0.16997254\n",
      "Iteration 19, loss = 0.16986993\n",
      "Iteration 20, loss = 0.16983531\n",
      "Iteration 21, loss = 0.16979688\n",
      "Iteration 22, loss = 0.16974538\n",
      "Iteration 23, loss = 0.16972036\n",
      "Iteration 24, loss = 0.16964230\n",
      "Iteration 25, loss = 0.16965639\n",
      "Iteration 26, loss = 0.16959202\n",
      "Iteration 27, loss = 0.16959428\n",
      "Iteration 28, loss = 0.16953742\n",
      "Iteration 29, loss = 0.16950189\n",
      "Iteration 30, loss = 0.16945177\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1017\n",
      "Iteration 1, loss = 0.18617809\n",
      "Iteration 2, loss = 0.17183498\n",
      "Iteration 3, loss = 0.17122368\n",
      "Iteration 4, loss = 0.17099633\n",
      "Iteration 5, loss = 0.17074071\n",
      "Iteration 6, loss = 0.17060289\n",
      "Iteration 7, loss = 0.17048015\n",
      "Iteration 8, loss = 0.17042316\n",
      "Iteration 9, loss = 0.17033165\n",
      "Iteration 10, loss = 0.17023361\n",
      "Iteration 11, loss = 0.17018109\n",
      "Iteration 12, loss = 0.17009550\n",
      "Iteration 13, loss = 0.17003665\n",
      "Iteration 14, loss = 0.17000801\n",
      "Iteration 15, loss = 0.16997591\n",
      "Iteration 16, loss = 0.16990965\n",
      "Iteration 17, loss = 0.16985951\n",
      "Iteration 18, loss = 0.16980350\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1018\n",
      "Iteration 1, loss = 0.18290917\n",
      "Iteration 2, loss = 0.17187834\n",
      "Iteration 3, loss = 0.17139273\n",
      "Iteration 4, loss = 0.17104365\n",
      "Iteration 5, loss = 0.17092743\n",
      "Iteration 6, loss = 0.17078984\n",
      "Iteration 7, loss = 0.17070126\n",
      "Iteration 8, loss = 0.17056822\n",
      "Iteration 9, loss = 0.17043201\n",
      "Iteration 10, loss = 0.17040691\n",
      "Iteration 11, loss = 0.17024892\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 12, loss = 0.17023575\n",
      "Iteration 13, loss = 0.17014700\n",
      "Iteration 14, loss = 0.17017069\n",
      "Iteration 15, loss = 0.17010353\n",
      "Iteration 16, loss = 0.17006761\n",
      "Iteration 17, loss = 0.16996633\n",
      "Iteration 18, loss = 0.17002188\n",
      "Iteration 19, loss = 0.16998802\n",
      "Iteration 20, loss = 0.16991175\n",
      "Iteration 21, loss = 0.16985111\n",
      "Iteration 22, loss = 0.16984778\n",
      "Iteration 23, loss = 0.16980473\n",
      "Iteration 24, loss = 0.16981069\n",
      "Iteration 25, loss = 0.16974122\n",
      "Iteration 26, loss = 0.16963696\n",
      "Iteration 27, loss = 0.16970887\n",
      "Iteration 28, loss = 0.16965395\n",
      "Iteration 29, loss = 0.16958346\n",
      "Iteration 30, loss = 0.16959817\n",
      "Iteration 31, loss = 0.16961791\n",
      "Iteration 32, loss = 0.16955215\n",
      "Iteration 33, loss = 0.16952206\n",
      "Iteration 34, loss = 0.16946453\n",
      "Iteration 35, loss = 0.16944990\n",
      "Iteration 36, loss = 0.16942375\n",
      "Iteration 37, loss = 0.16940545\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1019\n",
      "Iteration 1, loss = 0.21184052\n",
      "Iteration 2, loss = 0.17323735\n",
      "Iteration 3, loss = 0.17190695\n",
      "Iteration 4, loss = 0.17142556\n",
      "Iteration 5, loss = 0.17118205\n",
      "Iteration 6, loss = 0.17092740\n",
      "Iteration 7, loss = 0.17074567\n",
      "Iteration 8, loss = 0.17065206\n",
      "Iteration 9, loss = 0.17061277\n",
      "Iteration 10, loss = 0.17051038\n",
      "Iteration 11, loss = 0.17048918\n",
      "Iteration 12, loss = 0.17031009\n",
      "Iteration 13, loss = 0.17036307\n",
      "Iteration 14, loss = 0.17027753\n",
      "Iteration 15, loss = 0.17021870\n",
      "Iteration 16, loss = 0.17018179\n",
      "Iteration 17, loss = 0.17009383\n",
      "Iteration 18, loss = 0.17012516\n",
      "Iteration 19, loss = 0.17005576\n",
      "Iteration 20, loss = 0.17006864\n",
      "Iteration 21, loss = 0.16994993\n",
      "Iteration 22, loss = 0.16992253\n",
      "Iteration 23, loss = 0.16985354\n",
      "Iteration 24, loss = 0.16986122\n",
      "Iteration 25, loss = 0.16976985\n",
      "Iteration 26, loss = 0.16982703\n",
      "Iteration 27, loss = 0.16976707\n",
      "Iteration 28, loss = 0.16969677\n",
      "Iteration 29, loss = 0.16970196\n",
      "Iteration 30, loss = 0.16960369\n",
      "Iteration 31, loss = 0.16963562\n",
      "Iteration 32, loss = 0.16957539\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
      "1020\n",
      "Iteration 1, loss = 0.19670978\n",
      "Iteration 2, loss = 0.17236339\n",
      "Iteration 3, loss = 0.17165314\n",
      "Iteration 4, loss = 0.17131161\n",
      "Iteration 5, loss = 0.17105697\n",
      "Iteration 6, loss = 0.17092363\n",
      "Iteration 7, loss = 0.17077010\n",
      "Iteration 8, loss = 0.17068805\n",
      "Iteration 9, loss = 0.17062869\n",
      "Iteration 10, loss = 0.17058092\n",
      "Iteration 11, loss = 0.17046003\n",
      "Iteration 12, loss = 0.17037126\n",
      "Iteration 13, loss = 0.17031279\n",
      "Iteration 14, loss = 0.17026327\n",
      "Iteration 15, loss = 0.17023561\n",
      "Iteration 16, loss = 0.17013575\n",
      "Iteration 17, loss = 0.17014767\n",
      "Iteration 18, loss = 0.17008758\n",
      "Iteration 19, loss = 0.17003893\n",
      "Iteration 20, loss = 0.16999257\n",
      "Iteration 21, loss = 0.16993777\n",
      "Iteration 22, loss = 0.16989745\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n"
     ]
    }
   ],
   "source": [
    "all_preds = pd.DataFrame(index=df_ft.index)\n",
    "for seed_ in np.arange(1000, 1021):\n",
    "    print(seed_)\n",
    "    nn = neural_network.MLPClassifier(hidden_layer_sizes=(25, 10), verbose=1, random_state=seed_)\n",
    "    nn.fit(tr_f_sc, df_f[\"is_click\"])\n",
    "    all_preds[\"nn_seed_{}\".format(seed_)] = nn.predict_proba(te_f_sc)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90675, 21)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "1001\n",
      "1002\n",
      "1003\n",
      "1004\n",
      "1005\n",
      "1006\n",
      "1007\n",
      "1008\n",
      "1009\n",
      "1010\n",
      "1011\n",
      "1012\n",
      "1013\n",
      "1014\n",
      "1015\n",
      "1016\n",
      "1017\n",
      "1018\n",
      "1019\n",
      "1020\n"
     ]
    }
   ],
   "source": [
    "# all_preds = pd.DataFrame(index=df_ft.index)\n",
    "for seed_ in np.arange(1000, 1021):\n",
    "    print(seed_)\n",
    "    lgb = lightgbm.LGBMClassifier(n_estimators=650, max_depth=5, learning_rate=0.005, colsample_bytree=0.6, random_state=seed_)\n",
    "    lgb.fit(df_f[predictors], df_f[\"is_click\"])\n",
    "    all_preds[\"lgb_seed_{}\".format(seed_)] = lgb.predict_proba(df_ft[predictors])[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_cols = [col for col in all_preds if col.startswith(\"nn\")]\n",
    "lgb_cols = [col for col in all_preds if col.startswith(\"lgb\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90675, 42)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_t = hmean((all_preds[nn_cols].mean(1), all_preds[lgb_cols].mean(1).values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.DataFrame({\"impression_id\":df_ft.impression_id.values, \"is_click\":preds_t})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv(\"sub_2.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}