CNN_2D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "# TensorFlow and tf.keras\n",
    "# import tensorflow as tf\n",
    "# from tensorflow import keras\n",
    "# from tensorflow.keras import layers\n",
    "# from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Flatten, Dense, BatchNormalization, Dropout\n",
    "import pandas as pd\n",
    "\n",
    "import keras\n",
    "from keras import layers\n",
    "import keras.backend\n",
    "from keras.layers import Input, Conv2D, MaxPool2D, Flatten, Dense, BatchNormalization, Dropout\n",
    "import keras.models\n",
    "import keras.utils\n",
    "\n",
    "# Helper libraries\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import LogNorm\n",
    "import math\n",
    "import time\n",
    "import h5py\n",
    "import sklearn\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import data:\n",
    "Using jetImages_signal.npz and jetImages_bkg.npz to train the neural network and using jetImages_signal_test.npz and jetImages_bkg_test.npz for testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = 32\n",
    "# training data : 10,000 signal samples and 10,000 bkg samples\n",
    "traindata_sig = np.load('data/jetImages_signal.npz')\n",
    "traindata_bkg = np.load('data/jetImages_bkg.npz')\n",
    "trainimages = np.concatenate((traindata_sig['jetImages'], traindata_bkg['jetImages']),axis=0)\n",
    "trainlabels = np.concatenate((traindata_sig['labels'],traindata_bkg['labels']),axis=0)\n",
    "# testing data : 10,000 signal samples and 10,000 bkg samples\n",
    "testdata_sig = np.load('data/jetImages_signal_test.npz')\n",
    "testdata_bkg = np.load('data/jetImages_bkg_test.npz')\n",
    "testimages = np.concatenate((testdata_sig['jetImages'], testdata_bkg['jetImages']),axis=0)\n",
    "testlabels = np.concatenate((testdata_sig['labels'], testdata_bkg['labels']),axis=0)\n",
    "\n",
    "n_test = len(testlabels)\n",
    "n_train = len(trainlabels)\n",
    "trainimages = trainimages.reshape(n_train, grid, grid, 1)\n",
    "testimages = testimages.reshape(n_test, grid, grid, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print a plot of data to see what it looks like!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessing -> spit data into training and testing; take 80% for training and 20% for testing.\n",
    "\n",
    "If using CNN, uncomment \".reshape(n_data, grid, grid, 1)\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAD4CAYAAABmKcrZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df5CddZXn8fehCQlpQgLpJBASbH7EaAQMSy+4Mo5xxZm4jqLuqMCWizusqDVMaZW7Jf6okt0aS2ZXHbVktaJQYJWClD8zThx0KRFZGU2CKIEQDRClScwvCCSEJHRy9o97G/p2nnP63tv3d39eVbeSfs597vO9t2++eZ7ne77na+6OiEgvOabdDRARaTR1bCLSc9SxiUjPUccmIj1HHZuI9JxjW3mwgYEBHxwcbOUhRaaU9evX73L3eZN5jbPNfH+Vz90Gd7j7yskcrxla2rENDg6ybt26Vh5SZEoxsz9M9jX2A++r8rnXwcBkj9cMLe3YRKTzGd3fMXR7+0WkwY4Bjm93IyZJHZuIVDBgWruObfZy4IOULnHvdPcv1/M6GhUVkQqjl6LVPKp6PbObzGyHmW0Yt32lmW0ys81mdi2Au2909/cD7wSG6n0P6thEpMLoGVs1jyrdDFSMnJpZH3AD8EZgGXC5mS0rx94C3APcWe97mLDTNbMZwN3A9PLzv+3unzSzk4FvAYPAFuCd7v5UvQ2R5rKfxTF/bbKf3RUH/35FHNsQh8J23Fr7PtJ4NQ4eDJjZ2FSHVe6+auwT3P1uMxsct9+FwGZ3fxTAzG4DLgUecvfVwGoz+2fgm7W2H6pr/0Hg37v7PjObBtxjZj8C3k7pGvj68mnktcBH6mmEiHSOGu+x7XL3ei4ZTwMeH/PzMHCRma2g1LdMB9bU8bpAFR2bl+oa7Sv/OHoG6pR61xXl7bcAd6GOTaTr1TgqOtvMVgH/5O7/VMNhrGCbu/tdlPqSSanqHpuZ9ZnZ/cAO4Cfu/ktggbtvK7dmGzB/so0RkfZrwj22IsPA4jE/LwK2Tu4lX1TVpbS7HwaWm9kc4Htmdk61BzCzq4GrAU4//fS6GikirVXDPban3f3qOg6xFlhiZmcATwCXAVfU8TqFahoVdfc9lE4TVwLbzexUgPKfO4J9Vrn7kLsPzZs3qSlsItICNZ6xzTazVWb25vD1zG4F7gWWmtmwmV3l7iPANcAdwEbgdnd/sFHvoZpR0XnA8+6+x8yOBy4B/gFYDVwJXF/+8weNapSItE+No6ITnrG5++XB9jVMYoAgU037TwVuKeedHEOpZ/2hmd0L3G5mVwF/BN7RjAZOVfaH5+PgvyZ3N/7sQPH2GTPCXc5lbRhb5KeFse27nwlj02ccLNy+70/xnGn7w0gY4/7kPQ/GIZYXb9ZSH7EWDR40VTWjor8Fzi/Yvht4fTMaJSLtU2O6R7332JpKc0VFpEIvVPfQlCoRqdDowYN26PaOWUQarNGDB+2gjk1EKrSzbFGjqGOrgX0gCMxJdvrXJPaJJLYn/motetfvw9iup+cWbl960f3hPn3Eo5EXEJdyP3bu4TB2x7N/Wbj9lLMeC/fZvTseMe1bsTeMHbjn5DDGw3FIihlTYFRURKYWA6ZV2zOM6FJURLqAGRxbfcfWkdSxiUgFM5jW1+5WTI7SPUSkwugZWzUPlO4hIt3ADKZNr/rpuscmIl2gB6YedHnzG89OSoJ/H2xPqtOd8ImdYez4/ufCWJaCsZw4dWPH7OJ6n2fxSLjPVhaGsbOT/bazIIwt7y9u4yBbwn12zy1OVQHYxNIwNvNNcRt3UfyadueZ4T58Pg51TkJDE6ljE5Ge1OU9gwYPRKSSAX1VPjR4ICJdobZLUQ0eiEgXMEqL33UxdWwiUkmDByLSc9SxdSfLFsv6WhJbVLz5hHPilI6DB+Jz+gv614exBWwPY3PZHcYGKa6eMZ1D4T6LKxbkrnQ2m8PYYeJ5N1FsFnGVjsz5/DqM/fTw68LYcX3Fay9MWx6v1/D8fz0xjJldF8bc41jX6fIpVVOyYxORRBvP2MzsrcCbKC3AfoO7/7ie11G6h4hUGu3YqnlU83JmN5nZDjPbMG77SjPbZGabzexaAHf/vru/F3gP8K5634I6NhGpNDoqWs2jOjdTWmT9xUOUlvO8AXgjsAy43MyWjXnKJ8rxuqhjE5FKtZ2xDZjZujGPo3La3P1u4Mlxmy8ENrv7o+5+CLgNuNRK/gH4kbvfV+9b0D02EalU2z22Xe4+VMdRToOKUath4CLg74BLKM1oONvdv1LHa6tjE5FxRqdUVafeNQ+sYJu7+xeBL9bwOoUm7NjMbDHwdeAU4Aiwyt2/YKVx7/cCo7kOH3P3NZNtUEv8tzg045LxZ8wvWjy7OC1iJnGVjsX9cSrFHPaEsQF2hbG9zApj57KpcPt+Zob7ZLJUkKy6x0uDdhxKbsxElTgAdiTHek3f3WHsueB9PzZ3MNzn9/teGcZmH3h/GLPPhiH8w3Gs47RmVHQYWDzm50XA1ka9eDXNHwE+7O73mdksYL2Z/aQc+0d3/0yjGiMiHaC2KVX1zhVdCywxszOAJ4DLgCvqeJ1CEw4euPu20Zt47r4X2Ejp+lhEelFtgwcTVvcws1uBe4GlZjZsZle5+whwDXAHpT7ldnd/sFFvoaYTTjMbBM4HfglcDFxjZv8ZWEfprO6pRjVMRNqkwdU93P3yYPsaoCm3r6pO9zCzE4DvAB9y92eALwNnAcuBbUDhHQYzu3p0KHjnznjqkYh0kAaesbVDVf2ymU2j1Kl9w92/C+Du28fEvwr8sGhfd18FrAIYGhryyTZYRJqstlHR7qzHZmYG3AhsdPfPjdl+qrtvK//4NmBD0f4i0mWmSHWPi4F3Aw+Y2egKHR+jNAViOeDAFuB9TWlhnewDcWzG9XFKx+GR+CMZCKpqHJdUzoiqbUCe+pC9ZpRKkVmYjKTPZH8Yy9I9srSTg8F7O5S8r6zyx7EcDmNZ+3cxULg9ah/AGf9pSxi7++nXhDGSNFW7MY75VXGsLWobFa03j62pJuzY3P0eipPpuiNnTURqo9LgItJzeuBSVJPgRaRSg/PY2qHL+2URaYpeHxUVkSmmBy5Fu7z5ItJwWn6vvezfJsFvPx+GZp4QV+PYs2tOGIsWUckWPDmLR8LYfo4PYycllT+OT6qJRFVB+pJ0iWxRlmzhmFOTFJKoGkeW0rGH+LOfnyxuk71mlDaT7bOeC8LYq2f/Iow98NrzwtjOfaeHMftU8Xb/eLhLc9V2xtad6R4iMsUo3UNEek5tU6o6kjo2EamkwQMR6TkGzGh3IyZHCboiUmn0UrSahxJ062fRvOMPxfvMPiUezVvW91AYm7UgHi2LJpJnaxdkFrItjGUTu7MRvagtWRuzUdEFTz4dxvafHI+KRm2M1iCA/D1nE+6z9RyiUdHtzA/3OZcHwthWFoaxbJ2K3UPxeg5HNveHsbbQ4IGI9KQu7xm6vPki0nAaFRWRnqNRURHpOT0wpUqjoiJSqbayRY09tNmZZnajmX17Mq+jjk1EKjW4YzOzm8xsh5ltGLd9pZltMrPNZnYtgLs/6j75YundcSn6smD78niXOdPj9IZsQng9awNkE8X7GAljWdrGgmTS9/FJWkSU1jE9WWtg5uH49Sz+OBiYHb/vaH2I/X1xasZu4pSIp5IJ8o+zOIxFn0d2rGxyf5Z2MpLccV+2IE4x2vDX5xZut0/FWbJNnSDf+HtsNwNfAr7+wiHM+oAbgDcAw8BaM1vt7vEHVQOdsYnI0apP0B0YXTe4/Dgqp83d7wbGr6B0IbC5fIZ2CLgNuLRRze+OMzYRaZ3azth2uXuyPlfoNKhYAm0YuMjM5gKfAs43s4+6+6freG11bCIyTmuW3yta+c7dfTfw/hpep5A6NhGp1Jo8tmGouDm6CJKbmzVSxyYilVozV3QtsMTMzgCeAC4DrqjjdQpp8EBEKjV4+T0zuxW4F1hqZsNmdpW7jwDXAHcAG4Hb3f3BRr2FCftlM1tMaZj2FOAIsMrdv2BmJwPfAgaBLcA73f2pRjWswr7adxliXcObMciWwu1Z2sbcpOLDrOSNZakD2fGi2Mk7DoT7EBfwgEfj0Inz43UlPPhmHTc9Plhff5yGk6XoZKJUkJeyKdzndywNY1mqzRDrw1hWTeSYY5cVbj8yEO7SdN7A5ffc/fJg+xpgTW0tq041Z2wjwIfd/eXAq4C/NbNlwLXAne6+BLiz/LOIdDk/Bg7NqO5Bt9Zjc/dtUCoc5u57zWwjpaHaS4EV5afdAtwFfKQprRSRlnGDkb5q71Id6f56bGY2CJwP/BJYUO70cPdtZhafa4tI13AzDh9bbdcQz2hpp6oHD8zsBOA7wIfc/Zka9rt6NCt5586d9bRRRFrscF9fVQ+69VIUwMymUerUvuHu3y1v3m5mp5bP1k4FdhTt6+6rgFUAQ0ND3oA2i0gTOZaWix+nIy9FJzxjMzMDbgQ2uvvnxoRWA1eW/34l8IPGN09EWs0xRuir6tGpqjljuxh4N/CAmd1f3vYx4HrgdjO7Cvgj8I7JNMSKCxyU3Fa8+cJX/Czc5TEGw9hF/CqMZYueRNU9skVIsqoaWQWPLKUjWqAE4OStQVrH+CnIYz2bxOL1Zkq/9YAF36wZcVENsrye/f3Hh7EsK+Jw8BXfkaRfLK6YwlhpV1IV5FDye/kFrw5jgwu2FG5/dMsrwn2ayTEOVT+nqt4pVU1VzajoPRTP6wJ4fWObIyLt1guXoppSJSJHqaFj60iaUiUiFWq8x9a9o6IiMnWULkWr7hp0KSoina80eHBcu5sxKerYRKSCkxdh6AYd07H5A3HMflm8fdPBuArDm6fHI89ZKkUmSrPIFnPJUjqy/WY9G6c+zMhSN6JSfVlKR9yMNKWDk5PY7GB7vLYNM5LYSafHaTjH9tVe+WNPsjjMweRsZSbPhbHs8u1c4i/4DzcVZ0od86Hsl9afxCarpkvRjtTdrReRhqsx3aMjaVRURI5ymL6qHmhUVES6gRJ0RaTnOMbBGpap6kTq2ESkQi/cY+uOju2E4s2Lp8eTlbNfTDZpfWGyAlg9o6nZhPWZB5PJ8weTF81GMaMlBbJ9skXPNiaxbFQ0neweSL6NJ+6O11dgbvx72dVX3JBstDobMc3WsMhyv7KJ9S9Z+nAYi9gtLysODF5wQc0vNo46NhHpScpjE5GeUuOUqoYys37g/1CqOX6Xu3+jntdRuoeIVBi9FK0y3WNCZnaTme0wsw3jtq80s01mttnMRle5ezvwbXd/L/CWet+DOjYRqVAaFT2uqkeVbgZWjt1gZn3ADcAbgWXA5eVlPRfBCzck61tMFl2Kisg4NV6KDpjZ2NXJV5XXOXnx9dzvLq9wN9aFwGZ3fxTAzG6jtKTnMKXO7X4mceKljk1EjlLDqOgudx+q4xCnQcVQ8TBwEfBF4Etm9iag7lLjHdOxZWseHPN/iycDZ6kZe5lVVzv6krPf6NQ7Sx/JUkT6Ro6EMcvmP9czof2xZJ8s3eP+JJatJBt9jNnc7ezbmBzruANxKsis/uLP/zlmhvtkv88FxYuxAfA74qIMW1kYxqJUkHt+9oZwH74fbI9rBVStxnSPetc8KFpuwN39WeC/1PA6hTqmYxORztCiPLZhYPGYnxeR/xdbE3VsIlKhxilV9c4VXQssMbMzgCeAy4Ar6nidQhoVFZEKNaZ7TFjdw8xuBe4FlprZsJld5e4jwDXAHZTmt9zu7g826j3ojE1EjtLI6h7ufnmwfQ2wpsamVUVnbCJSQatUiUjPmRKrVJnZTcBfATvc/ZzytuuA9wI7y0/7WPm0MrX+frCTgmA0fA28esEvCrdfSLAYAnBskrZxFo+EscEkLyIa6l/87HC4z4xsnCfOHMhjjyaxbXXsk6R0XJekiVz34+Q1zwm2Z5/HGUksqU4yI86k4MyFfyrcvmBh/AHPnR4fbAuDYew1/DyMbU/yVR7gvMLth177s3CfXz382uLAfeEuNen26h7VXIrezLjpEGX/6O7Ly4+mXCeLSOuNLr9XzYNuvRQNpkOISI8avcdWpY68FJ3M4ME1Zvbb8sz96AITM7vazNaZ2TqO7IyeJiIdYvQeWzWPTlVvx/Zl4CxgOaW7Op+Nnujuq9x9yN2HOGZenYcTkVaakqtUufsLdZXN7KvADxvWIhFpqym7SpWZneruo+NvbwM2ZM8Xke5R4z22jlRNusetwApKdZeGgU8CK8xsOeDAFuB91RzsguWwbl1xzD4Q77f5tWcVbs8W5Jibrl4SyxbyWMZDhdv39x8f7rNgSZxWcMrsaOUVSKfqZbFoEZWsqkay8Mp1dyT7XZzETg+2J6kZvDyJZZVEktgzc6cVbt/aFzdkM8XfN4DdDISxrKLMliSX5ZHgeH+smCM+TvQvt6hmRo1Ko6JVzxWtt7pHU1UzKlo0HeLGJrRFRDrAlL0UFZHe1u0JuurYRKTClLjHJiJTSzuX32uU7m69iDTc6JSqbqayRSJSQWWLGuk9cWg6hwq3HxdsB5iTrGqR7ZcuvhJUDJnJc8mxDoaxdFGW7DczI4lFsnSP2UksStuAPHUjes3sWAeSWAu/qbPYF8bqTfeIUjoA9gcLy/zpkaTcySXBAjZf8HifGvR82SIRmVpatJhLU6ljE5EK6thEpCcp3UNEesoRjqllSlVHUscmIkfptEtRMzsT+Dgw293/eqLnd07H9u049IdTikeUlr5kU7hPNkF+JvvDWJa/8xzFk92zL0H6P19/MgyYzeEfSWLR4bLvafafczaa2spvT/aek9j+vuIRx4MtztPK1tnYxNLC7a88K17T4zeve1Vx4LHJz4Jv9D22onVTyttXAl+g9O38mrtfH7bJ/VHgKjNLeooXdU7HJiIdwWn4PbabgS8BXx/dYGZ9wA3AG4BhYK2ZrabUyX163P5/4+7Z8kZHUccmIuPUNKVqwMzGFiNb5e6rxj4hWDflQmBz+UwMM7sNuNTdP03p7G5S1LGJSIUaL0V3uftQHYc5DXh8zM/DwEXRk81sLvAp4Hwz+2i5AwypYxORCo7Vcg+y3kKTRTcDw2kT7r4beH+1L66OTUQqtKi6xzBUlAheRL6cdk3UsYnIUVpQQXctsMTMzgCeAC4DrqjjdQp1Tsf2Z419uWyi+66kyP+yJE0kEk1iBlh4uM7/hLKJ7tlk8Ujx/P2SZOkFHktiS5JYo/M7k5SO55OUlKgIQVRYAeK0HshThaYnBQ+y2Ap+Wrj9lkeSK69FwfbfxrtUq8Z7bBNeihatm+LuN5rZNcAdlEZCb3L3Byff+pLO6dhEpCM4xuEjjTtjC9ZNwd3XAGtqbF5V1LGJSAU/Yhw80OOrVInI1OJuHB7RKlUi0kucWjq2jqSOTUQquBsjzzdu8KAd1LGJyDjGkcM9Xhq8aGa+mZ0MfAsYBLYA73T3pybVktuS2JxphZv7XhLnMGxnfhg7O6m0kDk+WdsgcrAvuwkb1K2H+tc8iKYKZ+srJNOLf/v7OHZeNpEmWtugnlSVCUxLlpWIEk2z9QmitS0gT+3JXvMxBsPYtmjxiF1JpY5rgu3rgu21cKDLL0WrWaXqZmDluG3XAne6+xLgzvLPItILjhgcOLa6R7euUhXMzL+UUsIdwC3AXcBHGtguEWmnrP5dpe68FA0scPdtAO6+zczC6z4zuxq4GuD007O13ESkI5QKsnW1pi+Y7O6r3H3I3YfmzZvX7MOJyGSNdmzVPLr1UjSw3cxOLZ+tnUp6+1lEuoqTjmuN05GXovWesa0Griz//UrgB41pjoi0nQMHq3x0qGrSPY6amQ9cD9xuZlcBfwTeUc3B1g+D/fcgeFtclmDGV4pLGWSVFmaxt5om1WQvJxRun0Oc6bI/qRQB++JQPQu2ZOpcDOWJZLfzkuaHsiojWRpL8k31JNYXvLmo6kdpnyzdI/59ZqkgM5NUoeh4085+Jtzn0NwTC7fbw+vXhztVqwfusVUzKlo4Mx94fYPbIiKdYCp0bCIyxfRAx9b0UVER6TJTeFRURHrZFE3QFZFedYSmzOltJXVsIlKpB+6xtbRju2ARrPvfxTEbPi/c70BQseCh1y8L98kWc5mVpFnsYU4YG2B3sM9JybHitJNn5hZXLQE48ck6K39EVTWSBU/CfYBzk92C7Jf8eMVZCiVZQYmk/QeT9JcdLCjcfijJmcm+A9l+WQpJtjjKA88Wf8rP3xV/WLY5CJx2wQXhTtVSxyYiPUcdm4j0pA7r2MzsrcCbgPnADe7+4+z5SvcQkUq1pXtMyMxuMrMdZrZh3PaVZrbJzDabWVrT0d2/7+7vBd4DvGuiY+qMTUQqHYE6ikVnbga+BHx9dIOZ9QE3AG8AhoG1Zraa0p3WT4/b/2/cfbTQxifK+6XUsYlIJSef01tpwMzGDu+tcvdVFS9XXKz2QmCzuz8KYGa3AZe6+6cpLUVQwcyM0hz1H7n7fRM1Sh2biByt+ntsu9w9W/0ichrw+Jifh4GLkuf/HXAJpZkOZ7v7V7IX75yOLbnCnjH4ZOH2kTQ/IBZV6YC8CkO0WMd8tof7PJdUfOgbScrYnRyHsqIgYTZCvLZN+iVelH3VzkhiUQpJklqSpqQk39RDM+K0mahyRlaJI0vNyFI6otQSgIMcF8aO7y/+zu1b7uE+/h+LF3qxa1te3aPe5feK3kD4ht39i8AXq33xzunYRKQztCbdYxhYPObnRcDWRr24OjYRqVTblKp654quBZaY2RmUyv5dBlxRx+sUUrqHiBytgdU9ysVq7wWWmtmwmV3l7iOUVke9A9gI3O7uDzaq+TpjE5FKtV2KTnjGFhWrdfc1wJqa2lYlnbGJSKXRxVyqeageG6x/GOzfFcf83ng/W188RPjcBcWT0gEOJaNQxyZJOtuT4cNohC17vWz0bc/0eLJ1X3/xSDDAjHjwrb4FNrK1BpYnsWxUNPoYs1HRJHYgGSXe21c8Wg3xGgXRWggAB5OJ7o8Tr42brcGRfUcG2FW4fedIm9bhrS2PTfXYRKQLaBK8iPQcp5YpVfXmsTWVOjYRqaRLURHpOT1wKapRURGppFWqRKTnjKZ7VKf3LkXNbAuwl9IV+chEs/wveBmsC9I67KPJjiuLN+95Nk6X2NU/EMbmE08+35LkMETrF2xlYbjPwmT6WzaJf39/nIMx49lkvkvUlGRdgPRbkGUcZLEoPSNbJyFJO9nbH+8YFSeAeD2KbF2DbBL8c0H6yETtyGKPPT1YHCjOAilZmsQaofp7bB2pEWdsr3P37FcgIt2ktrmiGhUVkS7QA5eikx08cODHZrbezArfnJldbWbrzGzdzp07J3k4EWm60XSPah4darJnbBe7+1Yzmw/8xMwedve7xz6hXCZ4FcDQ0FBcOU9EOsdUTvdw963lP3cA36NUx1xEulmDV6lqh7o7NjPrN7NZo38H/gLYkO8lIh1vdPCgmkeHmsyl6ALge6XFYzgW+Ka7/0u9L+bjF9waw15evH3f5+eF+2z5y8Ewtpu5YWxxxfoS1cWmcyjcJ6uRf1yyX5ZywPw9YeikJ4u/bZatJ5BV93g6icVZLmG6hyfH2t8f/z/7eEUV6UrZ7zOyI6nisos4VShrR5a+E1WGATg8EvwzjH/NzdWaNQ+aqu6Orbxs1isb2BYR6RQNLDTZDkr3EJFKtaV7dCR1bCJSqbbqHh1JHZuIVOqB6h7q2ESk0hFqKTTZkdSxicjROuxS1MxeDnwQGADudPcvZ8/vjo7tsmB7UikiW1hjKZvCWJbu8RDLCrfP4alwn+eyxVySChOZw8mv7fDJxTkC0w/HaScnjiR3is+MQweyxVwC2/vjFJ16HUwW7tkdpG5sJ14RJ6vWkqXo9CXpHlnVmDlzi39nO689MdyHN8WhhmjgHCEzuwn4K2CHu58zZvtK4AtAH/A1d78+bI77RuD9ZnYM8NWJjqlCkyLSbDczrviYmfUBNwBvBJYBl5vZMjM718x+OO4xv7zPW4B7gDsnOmB3nLGJSKcaMLN1Y35eVZ4f/gJ3v9vMBsftdyGwuZwPi5ndBlzq7p+mdHZ3FHdfDaw2s38Gvpk1Sh2biEzGrokKzAZOg4r7PsPARdGTzWwF8HZKJVMnXD1eHZuIjFPTsGi9U6qsYFt4Z8/d7wLuqvbF1bGJyDgtmXowDBUTbxdBUke/RurYRGScmjJ0650ruhZYYmZnAE9Qyn24oo7XKdQdHVuUFZGkezz+bFyF4aX9cbrH3bwmjL0uOBPO0gN2JGkFL03STjJZ5Y9o0ZC5ffGyFPsXxikMpxyOy3vUk7qxL1nU5Kkk/SXbL/uM0yopdTiUpJb8mvPrasfOG4tXxfEHqm9XY9V0xjbhpaiZ3QqsoDTQMAx80t1vNLNrgDsopXvc5O4PTrrpZd3RsYlIC9XUsU14xubulwfb11DFQEA91LGJyDhOCwYPmkodm4iM05J7bE2ljk1Exun+gmyaUiUi49S0mstsM1tlZm9uT1uLdcUZm3+weLu9O95n30A8Yvfzz/x5GFve9+swtpmzCrcvYEe4z3y2h7FspC+b6D6LvWFsLsWjn7tZGu6TtfG4xfHk+W3JaHBU/z97X9k6BNl6AruSNQ/2cFKwPR6BzdZQ2MzZYSwbMR3+xpIw1nkaO3jQDl3RsYlIK3V/pUl1bCIyTkumVDWVOjYRGUeXoiLSk7r7UlSjoiIyzugZWzUPjYqKSFeY4peitdQsb4riUvEl58ShJx8+LYz97hXxWgmv4eeF2/cn6xocYnoYi9ZQAJifpJBkFgSpG1mt/mjiPOTtz9IsorSOx4mLE2TpEn3J6iL1TIJ/JEjdgfyz+s3/e1UYY/BAHNsSh/zjcaw9pvCo6Jia5W+gVFtprZmtdveHGtU4EWmH7l9/bzJnbIU1ywF1bCJdbWpPqSqqWX7UNZ6ZXW1m68xs3c6dOydxOBFpjZqmVHWkyXRsVdUsd/dV7j7k7kPz5jV+TUkRabSpPSra1JrlItIuU7tsUVNrlotIu3T/4IG517+WvZn9B+DzvFiz/FMTPH8n8IfyjwMQlKJoLbWjktpRqdva8RJ3n9Q9HzP7l/LxqrHL3VdO/LTWmlTHNqkDm62rc6FVtd7mlVQAAAMpSURBVEPtUDskpSlVItJz1LGJSM9pZ8e2qo3HHkvtqKR2VFI7ulDb7rGJiDSLLkVFpOeoYxORntOWjs3MVprZJjPbbGbXtqMN5XZsMbMHzOx+M1vXwuPeZGY7zGzDmG0nm9lPzOz35T+Ll1dqfjuuM7Mnyp/J/eVcxWa3Y7GZ/dTMNprZg2b2wfL2ln4mSTta+pmY2Qwz+5WZ/abcjv9R3t7y70i3avk9tnK5o98xptwRcHk7yh2Z2RZgyN1bmoBpZn8O7AO+7u7nlLf9L+BJd7++3Nmf5O4faUM7rgP2uftnmnnsce04FTjV3e8zs1nAeuCtwHto4WeStOOdtPAzMTMD+t19n5lNA+4BPgi8nRZ/R7pVO87YXih35O6HgNFyR1OGu98NPDlu86XALeW/30LpH1Q72tFy7r7N3e8r/30vsJFSpZiWfiZJO1rKS/aVf5xWfjht+I50q3Z0bFWVO2oRB35sZuvNrN0TeRe4+zYo/QODZPXg5rvGzH5bvlRt6eWOmQ0C5wO/pI2fybh2QIs/EzPrM7P7gR3AT9y9rZ9Ht2lHx1ZVuaMWudjd/w3wRuBvy5dmU92XgbOA5cA24LOtOrCZnQB8B/iQuz/TquNW0Y6Wfybuftjdl1OqmnOhmSXF7mW8dnRsHVPuyN23lv/cAXyP0mVyu2wv3+MZvddT36IHk+Tu28v/qI4AX6VFn0n5XtJ3gG+4+3fLm1v+mRS1o12fSfnYe4C7gJV0yHekG7SjY3uh3JGZHUep3NHqVjfCzPrLN4gxs37gL4AN+V5NtRq4svz3K4EftKMRo/9wyt5GCz6T8s3yG4GN7v65MaGWfiZRO1r9mZjZPDObU/778cAlwMN0yHekG7Rl5kGt5Y6a1IYzKZ2lQaku3Tdb1Q4zuxVYQak0zHbgk8D3gduB04E/Au9w96be2A/asYLSJZdTWlvpfaP3dZrYjj8Dfg48QKkYGMDHKN3fatlnkrTjclr4mZjZeZQGB/oonXzc7u7/08zm0uLvSLfSlCoR6TmaeSAiPUcdm4j0HHVsItJz1LGJSM9RxyYiPUcdm4j0HHVsItJz/j+KsImv/c4d9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plot input signal and background\n",
    "sig_trainimages = np.zeros_like(trainimages[1])\n",
    "bkg_trainimages = np.zeros_like(trainimages[1])\n",
    "for i in range(len(trainimages)):\n",
    "    if testlabels[i] == 1:\n",
    "        sig_trainimages += trainimages[i]\n",
    "    else:\n",
    "        bkg_trainimages += trainimages[i]\n",
    "#signal\n",
    "display_sig = sig_trainimages.reshape(grid,grid)\n",
    "plt.grid(False)\n",
    "plt.imshow(display_sig, interpolation='nearest', origin='low', cmap = 'jet', norm=LogNorm())\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build DNN:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAD4CAYAAABmKcrZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfV0lEQVR4nO3df5BdZZ3n8feXTtIJIZDfMb+Y5tewZmEIa1ewcN2Ni2LcYc1oDQgqy5TR4GgsmR1rQHa2ZF2ngFmEZQdkDJIKTCmQUVwys+wgsmYCOxRDohECMU7ECE1iOgkJSQgkpPPdP+5tp29zvk/fvr/vuZ9X1a10P98+5zz39O0n55zn+zyPuTsiInlyQrMrICJSa2rYRCR31LCJSO6oYROR3FHDJiK5M6aRB5s+fbr39PQ08pAiHWXjxo173H1GNfs408wPl/mzO+FRd19SzfHqoaENW09PDxs2bGjkIUU6ipn9qtp9HAauLvNnb4Dp1R6vHhrasIlI6zPav2Fo9/qLSI2dAExodiWqpIZNREoYMLbZlaiSGjYRKaFbURHJnY64YjOz8cB6oLv4899196+Y2VTgQaAH2A5c5u776lfVzvIF/jyM/QV/EsYu4O8zy3+y9/xwmzunfT6M7Ul0eh1OPIn5By7MLH+cS8Jtrub2MPZNvhjGpLbycMVWToLuEeDfuft5wEJgiZm9G7gOeNzdzwIeL34vIm1u8IqtnFfNj232TjP7SzP7rpn9YaX7GbFh84JDxW8H348DS4F7i+X3Ar9XaSVEpHUM9oqW8yqHma0ys34z2zysfImZbTWzbWZ2HYC7b3H3zwKXAb3VvIdyKtZlZpuAfuAxd38amOXuO4uV2QnMrLQSItI66nDFthooGZ1gZl3AncCHgAXAFWa2oBj7MPAkhTvBipTVsLn7gLsvBOYBi8zsnHIPYGbLzWyDmW3YvXt3pfUUkQYaU+YLmD749118LR++L3dfD7w6rHgRsM3dX3T3o8ADFO4Ccfe17n4h8Ilq6l82d99vZusotL67zGy2u+80s9kUruaytlkJrATo7e3VdL0iLW6UvaJ73L2SW8a5wMtDvu8DLjCzxcBHKXRWPlLBfoHyekVnAG8VG7UJwPuBm4G1wFXATcV/H660EiLSOhrUK2oZZe7u64B11e68nPrPBu4t3hOfAKxx9781s6eANWa2DHgJuLTaynSaP+LGMNbPnDC2io+HsYvpySz/L9O+Wna9hpqZfSE+ogW8kFl+Nw+G26x85e4wNm5unP6ynylh7K/4TBiTbA0aUtUHzB/y/TxgR612PmLD5u7PAm9LgnL3vcBFtaqIiLSGBiXoPgOcZWanAa8Al0Pif+xR0kSTIlJi8Fa0zM6Dkfdndj/wFHC2mfWZ2TJ3PwasAB4FtlC4E3y+Vu+h3ROMRaTGan3F5u5XBOWPUEUHQYoaNhEpkYchVe1efxGpsY4YBC8jSw1YP0p3GFvI1jC2ONHj3Z8Y5HE+m8JY5A1ODGPn8lwY25HouT3IpMzyI4nzsWZuZR3rGxIjb27lc5nl/4lvVHSsTmBookkRyRkDxpbbMhyrZ00qp4ZNREqYwRg1bCKSJ2YwtqvZtaiOGjYRKTGqK7YW1ebVF5FaM4OxcR9PW1DDJiKlcpDI1ubVb6wv8bXM8jeYHG7zLjaEsTEMhLFJHAxjs9gVxiZweNTHSpnG3lEfC2Ay+zPLuxJPm1NpJ4cTsffxozC2L/jdRGkgAM9ybhhbTcWzVbcPNWwikktt3jK0efVFpOYMUK+oiOSKbkVFJHcMEiPf2oIaNhEppSs2EckdNWz5Y4klaf5i6fbM8rMTs3SkZsCYxp4w1sMvw1gqLWL+wMuZ5bu6ZoXbzBqI00dO3vVWGPPxh8JY99SjmeUDiafSh3kjjM1MpLikzvGx4HiTEr+z/86XwljHUOeBiOSKrthEJHfUsIlI7qhXVERyR1dsIpI7athEJHc6YUiVmc0H7gPeARwHVrr77WZ2A/AZYHfxR68vrhPYEiYf+XUY+1j3g2Hs5qXbwliUqjCO7NQGgIX8JIydmEhvSKV0LHj1xTC2Y+rUzPLUbCEn741TOuiPQzYtjk3ufi2zvCsxlfQbJ8V1fLlrfhibT3aKC8CUYJaR1Gwh3+aTYezhYJEagG/x6TD2N1S2UE1TdMgV2zHgj939x2Y2CdhoZo8VY7e5+y31q56INFwndB64+05gZ/Hrg2a2BZhb74qJSJPk4IrthNH8sJn1AOcDTxeLVpjZs2a2ysym1LhuItIMgw1bOa8WVXbDZmYnAd8DrnH3A8BdwBnAQgpXdF8PtltuZhvMbMPu3buzfkREWk0nNGxmNpZCo/Ztd38IwN13ufuAux8H7gYWZW3r7ivdvdfde2fMmFGreotIvQz2ipbzalHl9IoacA+wxd1vHVI+u/j8DeAjwOb6VFFEGioHz9jKqf57gCuB58xsU7HseuAKM1sIOLAduLouNazQ57q/EcZSqQ/vY92oj3ViYlGT1GIoRxiX2C6e+cMSKRNz+1/NDrweb5OoIhxIxBLGR/+bH4m3Gft6nHYyf1ac0jHhULzdjlPi40X2Euex9DMzjH2YtWHsY4nPyINcVV7FGqVDekWfpPBWh2uZnDURqaEOuWITkU6ihk1EcicHDduo8thEpEM0qVfUzE43s3vM7LvV7EcNm4iUqnGCbjGBv9/MNg8rX2JmW81sm5ldB+DuL7r7smrfgho2ESk12Ctazqs8q4ElJYcw6wLuBD4ELKCQZbGg6roXtfWd9B9wVxh7X2Kxjh62V3S8KD0jtdDIpCNxakk8TwRM3Hs8DqZSN6JYIs2CNxOxHYlY6oMdbTeQ2CaRmnHykcSiMontxgQHnBzM+gGwj8lhbNFvRhO+3eO8P4wt4IUwFn2OV/OH4TZ1VeNnbO6+vjgcc6hFwDZ3fxHAzB4AlkLiRI2CrthEpNTobkWnDw6ZLL6Wl3mUuVAy31QfMNfMppnZXwLnm9mXK30LbX3FJiJ1MLqJJve4e2+FRxnO3X0v8NkK9ldCDZuIlGpMukcfMHT20HmkH3yMiho2ESllwPi6H+UZ4CwzOw14Bbgc+Hitdq5nbCJSqsaze5jZ/cBTwNlm1mdmy9z9GLACeBTYAqxx9+dr9Rba+ortaKJb7jATwtgOZoex3+G5MBYNdp8yEPewHe6O59af8ctDYYzsJQMKUj2cqQHttd7fyRUcK3UlkOrtjceeY4lP8dw3sycFeGtivM3hU+LfWWqA/LmJz060XgakJ2Voitr3il4RlD9Cncact3XDJiJ10uYtQ5tXX0RqrhOW3xORDpODQfBtXn0RqblOmGhSRDqMrthEJHfUsDWG3ZNd/o1l68Nt5rAzjJ3JtjB2JHENPp/sefdP3hUP0OZAIpbI9kgOFk+lgkTrIQRLIYwotV3q0zM1KE+ldMSZFOn3nBL8Og/OifNOZg3EkxqM64pzY/Zwbhg7mJjyIIr9B/463OZvuDSMVU0Nm4jkknpFRSRXdMUmIrmjXlERyR1dsYlI7qhhE5Hc6YSGzczmA/cB7wCOAyvd/XYzmwo8CPQA24HL3H1fPSp5w7JrM8vfIJ6FoTsxZUVXIpfizIE4FWTcm0HqRmp2jErWBYA4bQPSM25E6RSpNItKj5V6b9E+E+sT0J+IJWbjSNY/2G7qjnihh9enJWbzSvQWzkq8gZdL5lQcvsvsz+N7idOZqGe6B+Bt3itaznxsx4A/dvd3Au8GPl9cTeY64HF3Pwt4vPi9iLQ5PwGOji/v1apGvGJz951QyHZ194NmtoXCQgxLgcXFH7sXWAdkX1qJSNtwg2Nd5c5Bm1hNrYlGdSddXELrfOBpYFax0cPdd5pZYipAEWkXbsbAmHKbhqN1rUulym7YzOwk4HvANe5+wCxrkZnM7ZYDywFOPfXUSuooIg020NXeD9nKut40s7EUGrVvu/tDxeJdZja7GJ9N8OjX3Ve6e6+7986YMaMWdRaROnKMAbrKerWqERs2K1ya3QNscfdbh4TWAlcVv74KeLj21RORRnOMY3SV9WpV5dyKvge4EnjOzDYVy64HbgLWmNky4CWq7H++krvD2Pwgr+Aifhhu052490+lexzpinMYTn6tgnSPeKKI9OweqfSMAxXsM1XH1Awelc7uEaV1pGbpmJWIpf6GUu8t6rlLnN+uU+IH4t3d8edqZuKXfRrbw1g0u0cqncmeDgL/4l3vCjcqk2PJhZLaQTm9ok+SvWozwEW1rY6INNvgrWg7a/P8YhGpBzVsIpIrg8/Y2pkaNhEpUbgVbe+mob1rLyI1V+g8GNfsalRFDZuIlHDQrWitTCaeGGRW0I2eulwex8Ew1pWYDmL6q4kcjChLJJ4oovK0jdR2qRk3otSHSvf3UiJWyYwbqW0qWaSmDsYn/ireMSau5CvzoxVs0g1FLxszy5/gveE2fkF2uf1sY/bORkW3oiKSM0r3EJFcUsMmIrmiKzYRyR3HkguHtwM1bCJSQldsNXTHT/8kjK0878rM8tTJH5MY6J4aBG+pHs6oZy7Vm5caoJ06ViqW2mdUl1QdU72iqXUZUj2c0SD41HSkqammUxcQqR7fqB6p85v4q3h9ZjwhTqrn82x+Hsai3s+FbMosB/gY92aWT3lXT00GwathE5HcUR6biORKKw6pMrPTgf8MnOLuvz/Sz5e7YoOIdIhaz6BrZqvMrN/MNg8rX2JmW81sm5klV7lz9xfdfVm576G1mmURabpCr2hNx4quBu6gsD4xAGbWBdwJfADoA54xs7UUphS9cdj2n3L31Kqzb6OGTURKjPJWdLqZbRjy/Up3X1myP/f1xRXuhloEbHP3FwHM7AFgqbvfCFxSUcWHUMMmIm8zil7RPe7eW8Eh5gIvD/m+DwhGwIKZTQP+DDjfzL5cbABDLdOw3XBevNbytCAfYT+Tw216+GUYmzKwP65I6ozEWSK1V+mg7yiNIZXukbjI74tPI/NS6R6RVNpGpekeUUoHxKkxqWMlUkEmvhavh3BkZjzxws5gXQOAM/hFZvnP+e1wm0ePfDCz/FDJekuVaVC6R9ZyAx79sLvvBT5b7s5bpmETkdbQoIatD5g/5Pt5pLMmR0UNm4iUaNCQqmeAs8zsNOAV4HLg47XaudI9RKREHdI97geeAs42sz4zW+bux4AVwKPAFmCNuz9fq/egKzYReZta3oq6+xVB+SPAIzU70BBq2ESkhFapEpHcacUhVaM1Yu3NbBWFhLl+dz+nWHYD8Blgd/HHri9eVib1sZMv8bXMWE9iioloNo4LeDrcZtKRuOt9Yn/cZR8sr1AQpUWkcqJTsUrXIUjtM9ou0d+095/i2LcSh/r05jg2L3pvqfeciiWWokhuFy1DkEoRSaXaxMsaMDWRJ9J9yvb4cN3ZV0ezE7+0G7uzRyDdZK+E24xGJ8zusZphwyGKbnP3W2peIxFpqo5Yfi8YDiEiOZWHZ2zVpHusMLNniyP3p0Q/ZGbLzWyDmW04vDt1zyAirWDwGVs5r1ZVacN2F3AGsBDYCXw9+kF3X+nuve7ee+KMSsbgiEij1TKPrRkqanLd/TeP2M3sbuBva1YjEWmqjp0a3Mxmu/vO4rcfARL9YyLSTvLwjK2cdI/7gcUU5l3qA74CLDazhRRG428Hri7nYPOYzS38aWbs09wRbncwmBmhK9Evv6D7hTA2fU6cSzFp4lthzKIZIVIzRaTSClIzblSyUEoqlthmWiL26TijhnnnJOoxJyhPLeYSbQMwLRFLpGBwalCe+uQn6vHW7Dh28JT4g/ACC8LY0WBc5n7CR9c8zvszyw+wPtymXIVe0ZwvvxcMh7inDnURkRbQsbeiIpJvathEJFc64hmbiHSWjhgrKiKdpSOGVIlIZ9GtaA39NlvD2KxgOovpySkwKnMk0cs9PjpbqdSMxMIgybOf+lyleuKjuqTqkUjBmDc/jlWUnpFKVUmdj0oXgYkyglK/s8Sxjo6PB+uMGYhX+5neFX9Wn+PczPLUYkW7mJVZfqxGf9K6FRWRXFG6h4jkjho2EcklPWMTkVw5zgn5H1IlIp1Ht6I1cvNA9hzuAGd2bcssv5abKzpWd9fRMDZ5zP54w5ODtRJSPY6p3rdXE7GTErGUqGMu9ZtOxaJB5JAe0B6971SvaGowe+o8VjLNX4W9ogNj4j/4XV3ZPZUAOxJdyHuCLuTUwPknf/qB7MAbJ4fblEvP2EQkdxw9YxOR3NGQKhHJGd2KikjuOMYRjRUVkTzR7B4ikku6Fa2RC7riyfU/xoOZ5dFaCACTidM2UpfZqe58H5+d7mGp1IHUUqqp9IZKB8HH47ArO1aq/nF2Q1zHStdySA10T4n2WeFKkHu64sUXDjMhjKWmAZrPy5nlqTUP/Lzscnth48ZwozLpGZuI5I5jDBxXwyYiOeLHjSNvakiViOSIuzFwTFdsIpInjho2EckXd+PYW63VsJnZO4EvAtOBx939rtTPx/Mci0iHMo4PjCnrVdbezFaZWb+ZbR5WvsTMtprZNjOLZ8EA3H2Lu38WuAzoHemYI9bMzFYBlwD97n5OsWwq8CDQA2wHLnP3fSPtK+UnLAxj5/JcZvmZZM/6AfGMCRCvoQBwpCt+aDqu+63M8mQmQq3n6q90u9R/wKnUh9cSsUpn44ik0j0qTROpoB6/npPaYaybeNaYcYnYnmDmj0kcrKgeVXOgtreiq4E7gPsGC8ysC7gT+ADQBzxjZmspfFJvHLb9p9y938w+DFxX3FdSOVdsq4Elw8quo3A5eBbwePF7EcmD4wZvjinvBdPNbMOQ1/Lhu3P39bx9kq5FwDZ3f9HdjwIPAEvd/Tl3v2TYq7+4n7XufiHwiZHewohXbO6+3sx6hhUvBRYXv74XWAdcO9K+RKRNpO4YSu1x9xFvDTPMhZLM5D7gguiHzWwx8FEK90CPjLTzSjsPZrn7TgB332lm4ZSDxRZ8OcCpp6ZmLRSRllCYkK3eLDhyJndfR+ECqix17zxw95Xu3uvuvTNmzKj34USkWoMNWzmvyvUBQ1etnQfsqGqPQ1TasO0ys9kAxX/jp/Ei0l4ceKvMV+WeAc4ys9PMbBxwObC2qj0OUWnDtha4qvj1VcDDtamOiDSdA0fKfJXBzO4HngLONrM+M1vm7seAFcCjwBZgjbs/X6u3UE66x/0UOgqmm1kf8BXgJmCNmS0DXgIurbYiOzk9rsMr2f0S18y9LdwmmjEB0ukeqVkYDk7MXmGla+ahcJuxlcy2AZWnYEQLy6RSRFKL0aRm90gt5hJ9slLpL6lPY6r+laR7VDgUMrUoS6194ZvfCmMrrq7jgWv8jM3drwjKH6GMjoBKlNMrmlkp4KIa10VEWkFjOg/qSkOqRKSUGjYRyR01bCKSS2rYRCRXjpPuUGoDathEpJRuRRvD52b39X8hMWXFPiaHsR8mOnQv5B/CWDTbQtcp8adg6oHEf32plI69iVhqu+g3WmlKRyqWSrOI0imyM2YKKp2BJJW6EWx3YNrYcJPUQiZjEqvlpGaUSfkm2bkbXs+UjhQ1bCKSO2rYRCSX1LCJSK7oik1Ecuc48EazK1EdNWwiUsoh0UfSFtSwicjb6Va0eV5gQRhLLayxmB+FsTc4cdT1mMz+MHZgTpxWMOm1eEIrS32wKlkoJTUjSCqlIzU1TWp2j0iq7qlUkERKx5uJehwdn33+93XF6UB7mR7GUikdqc9jKk1kTu3mV6wNPWMTkdxRwyYiuaMhVSKSS7piE5Fc0a2oiOTO4GIubaytG7b/+95LwtjeJ+KerR62h7GBxCmJBsEnt+nK3gbg8NQ49o5jcTfmm4mexa7gf9qxqTUDKu0xnZWIRePIU5+4xPt6K9Erenhi/OaOBt2ph5gUbrM/MYHCLzgzsd2UMHbj1q+GMT87DDWH8thEJHd0KyoiueNoSJWI5IxuRUUkd3QrKiK5o4ZNRHKn09M9zGw7cJDCHfkxd++tRaXK5U/EMfvf7w5j6353Wxi7iB+GsdOCEeGHmRBuk5o/PzV4fvfMeET4iUcOx8cLfqMHTxkXbjNlYjx+xipNzzg5uzyq30j2Tpwaxg4mUjei2N7EYPb/x4VhLDVg/VEuDmOnn/18GIN/mYg1iZ6x8T5331OD/YhIK9BYURHJnRzcip5Q5fYO/MDMNprZ8qwfMLPlZrbBzDbs3r27ysOJSN0NpnuU82pR1V6xvcfdd5jZTOAxM/uZu68f+gPuvhJYCdDb2+tVHk9EGqHNe0WrumJz9x3Ff/uB7wOLalEpEWmiwXSPcl4tquKGzcwmmtmkwa+Bi4HNtaqYiDTJYOdBOa8WVc2t6Czg+2Y2uJ/vuPvf1aRWNeC/G8fsjz4ZxibcFg+Si+a0v4B/DLeZTtxhPC6xoMCJicF6B7vjVJBorYcJxCkiL02dEcZ+67X4ueibcQYGA2Oy/8882B2nZpw4ENdxB3PC2BHiVJZoDYt9iRk8UrO1fO7m1WHspBXxuTo4MT7H9qHscv8/4Sb11ckJuu7+InBeDesiIq2ixRo2M1sM/DfgeeABd1+X+vlqe0VFJG8G0z3KeZXBzFaZWb+ZbR5WvsTMtprZNjO7roxaHQLGA30jHVN5bCJSqvaze6wG7gDuGywwsy7gTuADFBqqZ8xsLYUpSm8ctv2ngCfc/e/NbBZwK/CJ1AHVsIlIqdE9Y5tuZhuGfL+ymOL1z7tzX29mPcO2WwRsKz7SwsweAJa6+41APDU27CO5ymyBGjYRKXWc0Uw0uafCMeJzgZeHfN8HXBD9sJl9FPggMJnC1V+SGjYRebv6jyqwjLIwgd/dHwIeKnfnHdmw+W1x7EvsCmMvMz+zPDWDx1bilTpSs3tExwKYzt4wFi04k5oBI5V28vqcuH9pe3dPGDsWnJOdibSNcV3ZqSqQrv+0REpNdP6PJlJEXuCdYYxPxslbqZSOlKaldaTUf4xQH5R8yOdBYuqUUVKvqIg0wzPAWWZ2mpmNAy4H1tZq52rYRKSuzOx+4CngbDPrM7Nl7n4MWAE8CmwB1rh7atK6UenIW1ERaRx3vyIofwR4pB7HVMMmIsOMrlu0FalhE5Fh2n+mSTVsIjJM+4+CV8M2zC38aRizm7PL17z7qnCbS/7tX4exrkSy0JnEC87sYlYYi9JEjiSSteeX5EmWeqM7e3YMgD1MD2PdQQrJuGD2EYD9iRk3Uuke/cwcdT3+J18It/l5am6HuXEoP3TFJiK5o4ZNRHLHUeeBiOSMnrGJSO7oVlREckdXbJ1lQ3axX5va6NIwcjs/DGOf584w9h//eb6+t5kWDJBPDfo+MbEewi84I4wdDtYTANhOT2b5zMQkA7PoD2Pf4eNh7AdXLg1jXBOU/zrehMR6GZ1BV2wikju6YhOR3NGQKhHJHd2Kikgu6VZURHJFV2wikjsd3rCZ2RLgdgprAX7L3W+qSa1a1XdvyCy2L2eXA+mlXb+WWGzn98eGoRu+FozGBy7+4MOZ5YtZF27zAgvC2CYWhrF9iUHr57Mps/zaX90absOxxMdxU9baHwX+V/FmUokO7hWNFjx19xdqVTkRaYbO7hXNXPAUUMMm0tba/1a0msVcshY8fdtsVWa23Mw2mNmG3bt3V3E4EWmMwVvRcl6tqZqGrawFT919pbv3unvvjBmVrbsoIo00eMVWzqs1VXMrWtcFT0WkWTq484AhC54Cr1BY8DQeqSwibaL9Ow/MvfK17M3s3wP/g0K6xyp3/7MRfn438Kvit9OBPRUfvHZUj1KqR6l2q8dvuXtVz3zM7O+KxyvHHndfUs3x6qGqhq2qA5ttcPfephxc9VA9VI9cq6bzQESkJalhE5HcaWbDtrKJxx5K9SilepRSPdpQ056xiYjUi25FRSR31LCJSO40pWEzsyVmttXMtpnZdc2oQ7Ee283sOTPbZGbBGlR1Oe4qM+s3s81Dyqaa2WNm9k/Ff6c0qR43mNkrxXOyqZirWO96zDezH5nZFjN73sy+WCxv6DlJ1KOh58TMxpvZP5rZT4v1+K/F8oZ/RtpVw5+xFac7+jlDpjsCrmjGdEdmth3odfeGJmCa2b8BDgH3ufs5xbI/B15195uKjf0U9/TCfnWqxw3AIXe/pZ7HHlaP2cBsd/+xmU0CNgK/B/wBDTwniXpcRgPPiZkZMNHdD5nZWOBJ4IvAR2nwZ6RdNeOK7TfTHbn7UWBwuqOO4e7rgVeHFS8F7i1+fS+FP6hm1KPh3H2nu/+4+PVBYAuFmWIaek4S9WgoLzhU/HZs8eU04TPSrprRsJU13VGDOPADM9toZsubVIdBs9x9JxT+wICZTazLCjN7tnir2tDbHTPrAc4HnqaJ52RYPaDB58TMusxsE9APPObuTT0f7aYZDVtZ0x01yHvc/V8BHwI+X7w163R3AWcAC4GdwNcbdWAzOwn4HnCNux9o1HHLqEfDz4m7D7j7Qgqz5iwys3Pqfcw8aUbD1jLTHbn7juK//cD3KdwmN8uu4jOewWc9/c2ohLvvKv5RHQfupkHnpPgs6XvAt939oWJxw89JVj2adU6Kx94PrAOW0CKfkXbQjIbtN9Mdmdk4CtMdrW10JcxsYvEBMWY2EbgY2Jzeqq7WAlcVv74KyF6Vpc4G/3CKPkIDzknxYfk9wBZ3H7raS0PPSVSPRp8TM5thZpOLX08A3g/8jBb5jLSDpow8GO10R3Wqw+kUrtKgMC/ddxpVDzO7H1hMYWqYXcBXgP8FrAFOBV4CLnX3uj7YD+qxmMItlwPbgasHn+vUsR7/GngCeI7CZGAA11N4vtWwc5KoxxU08JyY2e9Q6BzoonDxscbdv2pm02jwZ6RdaUiViOSORh6ISO6oYROR3FHDJiK5o4ZNRHJHDZuI5I4aNhHJHTVsIpI7/x9tquEjXqVVAgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#background\n",
    "display_bkg = bkg_trainimages.reshape(grid,grid)\n",
    "plt.grid(False)\n",
    "plt.imshow(display_bkg, interpolation='nearest', origin='low', cmap = 'jet', norm=LogNorm())\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    model = keras.Sequential([\n",
    "        Conv2D(32, (11, 11), activation='relu',\n",
    "               input_shape=(grid, grid, 1)),\n",
    "        Conv2D(32, (3, 3), activation='relu'),\n",
    "        MaxPool2D((2, 2)),\n",
    "#         Dropout(0.2),\n",
    "        Conv2D(32, (3, 3), activation='relu'),\n",
    "        Conv2D(32, (3, 3), activation='relu'),\n",
    "        MaxPool2D((2, 2)),\n",
    "#         Dropout(0.2),\n",
    "        Flatten(),\n",
    "        #custom input layer\n",
    "        Dense(32, activation = 'relu'),\n",
    "        Dense(64, activation = 'relu'),\n",
    "#         Dropout(0.2),\n",
    "        Dense(1, activation='sigmoid')])\n",
    "\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                optimizer='adam',\n",
    "                metrics=['mean_squared_error', 'binary_crossentropy', 'accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "CNN = build_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test model's prediction $before$ training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10/10 [==============================] - 0s 3ms/step\n",
      "[[0.5001028 ]\n",
      " [0.5002982 ]\n",
      " [0.50034815]\n",
      " [0.49996698]\n",
      " [0.5002648 ]\n",
      " [0.5002786 ]\n",
      " [0.5001765 ]\n",
      " [0.5000862 ]\n",
      " [0.5001077 ]\n",
      " [0.50013745]]\n",
      "[0.6927937269210815, 0.24982330203056335, 0.6927937269210815, 0.8999999761581421]\n"
     ]
    }
   ],
   "source": [
    "example_batch = trainimages[:10]\n",
    "example_labels = trainlabels[:10]\n",
    "example_result = CNN.predict(example_batch)\n",
    "results = CNN.evaluate(example_batch, example_labels)\n",
    "print(example_result)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now train! (warning: if building CNN, computer tends to get loud)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.18541, saving model to model/CNN_test.h5\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.18541 to 0.05203, saving model to model/CNN_test.h5\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.05203 to 0.02723, saving model to model/CNN_test.h5\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.02723 to 0.00928, saving model to model/CNN_test.h5\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.00928\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.00928\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.00928\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.00928 to 0.00622, saving model to model/CNN_test.h5\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.00622\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.00622\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.00622 to 0.00439, saving model to model/CNN_test.h5\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00020: val_loss did not improve from 0.00439\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.00439\n"
     ]
    }
   ],
   "source": [
    "\n",
    "    \n",
    "checkpoint_path = \"model/CNN_test.h5\"\n",
    "# if not os.path.exists(\"model\"):\n",
    "#     os.mkdir(\"model\")\n",
    "\n",
    "# Create checkpoint callback\n",
    "# cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, \n",
    "#                                                  save_best_only=True,\n",
    "#                                                  verbose=1)\n",
    "model_checkpoint = keras.callbacks.ModelCheckpoint(checkpoint_path, monitor='val_loss', \n",
    "                                   verbose=1, save_best_only=True, \n",
    "                                   save_weights_only=False, mode='auto', \n",
    "                                   period=1)    \n",
    "EPOCHS = 1000\n",
    "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n",
    "history = CNN.fit(\n",
    "  trainimages, trainlabels,\n",
    "  epochs=EPOCHS, validation_split = 0.2, verbose = 0,\n",
    "  callbacks=[early_stop, model_checkpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now make a plot that shows the pregression of accuracy through each training epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAa7UlEQVR4nO3dfZQddZ3n8fcnTWIHQgh5IpEQA0xGDEFC02YDZCUQZgExBBUGwoMR2JPDKAJmHY3IKOY4c+KgLjDOLmY0magsERwesiwgkOHhMEowYOQhERIVsKFNQhwSUZ46fPePW13eNPd2V3e6qpruz+uce27Vr6ru73vr3r7f/tWv6leKCMzMzAAGlR2AmZn1HU4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmqdySgqRlkrZIerKqbKSkeyRtTJ73rVr2BUmbJD0t6cS84jIzs/rybCn8K3BSh7JFwOqImAysTuaRNAU4Czg02eZ/SWrIMTYzM6sht6QQEQ8Cv+9QPBdYkUyvAE6rKl8ZEa9HxG+ATcD0vGIzM7Pa9ii4vv0iohUgIloljU3K9wcerlqvJSl7G0kLgAUAe+2115GHHHJIjuGamfU/jz766EsRMabWsqKTQj2qUVZz/I2IWAosBWhubo61a9fmGZeZWb8j6bl6y4o++2izpPEAyfOWpLwFOKBqvQnAiwXHZmY24BWdFFYB85Pp+cBtVeVnSXqXpAOBycAjBcdmZjbg5Xb4SNINwCxgtKQW4MvAEuBGSRcCzwNnAETEU5JuBNYDbcCnImJnXrGZmVltuSWFiJhXZ9HsOuv/PfD3ecVjZn3Pm2++SUtLC6+99lrZofRLjY2NTJgwgcGDB2fepq90NJvZANTS0sLee+/NpEmTkGqdb2I9FRFs27aNlpYWDjzwwMzbeZgLMyvNa6+9xqhRo5wQciCJUaNGdbsV5qRgZqVyQshPT/atk4KZmaWcFMxswNq2bRvTpk1j2rRpjBs3jv333z+df+ONNzK9xvnnn8/TTz+duc7vfOc7jBkzJq1n2rRp3do+b+5oNrMBa9SoUaxbtw6AK6+8kmHDhvHZz352l3Uigohg0KDa/0MvX7682/Wec845XH311XWXt7W1sccef/557iqGajt37qShoefjibqlYGbWwaZNm5g6dSoXXXQRTU1NtLa2smDBApqbmzn00ENZvHhxuu7MmTNZt24dbW1tjBgxgkWLFnH44Ydz1FFHsWXLlk5q2dW9997LCSecwFlnncURRxxRM4Yf/OAHHHbYYUydOpXLL78cIK33iiuuYPr06TzyyO5d9+uWgpn1CV/5v0+x/sUdvfqaU949nC/PObRH265fv57ly5dz3XXXAbBkyRJGjhxJW1sbxx13HKeffjpTpkzZZZvt27dz7LHHsmTJEhYuXMiyZctYtGjR2177+uuv5/7770/n23/IH374YdavX8/EiRPZtGnTLjG0tLRwxRVXsHbtWvbZZx9OOOEEbr/9dk466SS2b99OU1MTX/3qV3v0Xqu5pWBmVsPBBx/MBz7wgXT+hhtuoKmpiaamJjZs2MD69evfts3QoUM5+eSTATjyyCN59tlna772Oeecw7p169LHkCFDADjqqKOYOHFizRjWrFnD8ccfz+jRoxk8eDBnn302Dz74IABDhgzhIx/5SK+8b7cUzKxP6Ol/9HnZa6+90umNGzdyzTXX8MgjjzBixAjOPffcmuf/t/+4AzQ0NNDW1tbjOjvOR9QcOBqoJKPeOrXXLQUzsy7s2LGDvffem+HDh9Pa2sqPf/zjwmOYMWMG9913H9u2baOtrY2VK1dy7LHH9no9bimYmXWhqamJKVOmMHXqVA466CCOOeaY3Xq9jn0K3/72t7vcZsKECSxevJhZs2YREcyZM4dTTjml262RrqizJklf55vsmL2zbdiwgfe9731lh9Gv1drHkh6NiOZa6/vwkZmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZjZgzZo1620Xol199dV88pOf7HS7YcOG1SxvaGjYZUjsJUuW9FqsRfHFa2Y2YM2bN4+VK1dy4oknpmUrV67kqquu6tHrDR06NB2Ku56OQ1t3HCa7nqzr7S63FMxswDr99NO5/fbbef311wF49tlnefHFF5k5cyavvPIKs2fPpqmpicMOO4zbbrutx/VMmjSJxYsXM3PmTG666SZmzZrF5ZdfzrHHHss111zDc889x+zZs3n/+9/P7Nmzef755wH4xCc+wcKFCznuuOP4/Oc/3yvvuStuKZhZ33DnIvjdE737muMOg5PrH8IZNWoU06dP56677mLu3LmsXLmSM888E0k0NjZyyy23MHz4cF566SVmzJjBqaee2unAc6+++irTpk1L57/whS9w5plnAtDY2MhDDz0EwHXXXcfLL7/MAw88AMCcOXP4+Mc/zvz581m2bBmXXHIJt956KwDPPPMM9957727dOKc7nBTMbEBrP4TUnhSWLVsGVEYlvfzyy3nwwQcZNGgQL7zwAps3b2bcuHF1X6uzw0ftyaHW/E9/+lNuvvlmAM477zw+97nPpcvOOOOMwhICOCmYWV/RyX/0eTrttNNYuHAhjz32GK+++ipNTU1AZdC6rVu38uijjzJ48GAmTZpUc7jsrDobFruj6tZIZ+vlwX0KZjagDRs2jFmzZnHBBRcwb968tHz79u2MHTuWwYMHc9999/Hcc8/lFsPRRx/NypUrgUoymjlzZm51dcUtBTMb8ObNm8dHP/rR9IcZKndHmzNnDs3NzUybNo1DDjmky9fp2Kdw0kknZTot9dprr+WCCy7gqquuYsyYMSxfvrxnb6QXeOhsMyuNh87On4fONjOzHnNSMDOzlJOCmZXqnXwIu6/ryb51UjCz0jQ2NrJt2zYnhhxEBNu2baOxsbFb2/nsIzMrzYQJE2hpaWHr1q1lh9IvNTY2MmHChG5t46RgZqUZPHgwBx54YNlhWBUfPjIzs1QpSUHSZyQ9JelJSTdIapQ0UtI9kjYmz/uWEZuZ2UBWeFKQtD9wCdAcEVOBBuAsYBGwOiImA6uTeTMzK1BZh4/2AIZK2gPYE3gRmAusSJavAE4rKTYzswGr8KQQES8AXweeB1qB7RFxN7BfRLQm67QCY2ttL2mBpLWS1vqMBTOz3lXG4aN9qbQKDgTeDewl6dys20fE0ohojojmMWPG5BWmmdmAVMbhoxOA30TE1oh4E7gZOBrYLGk8QPK8pYTYzMwGtDKSwvPADEl7qnInidnABmAVMD9ZZz7Q8xuimplZjxR+8VpErJH0I+AxoA34ObAUGAbcKOlCKonjjKJjMzMb6Eq5ojkivgx8uUPx61RaDWZmVhJf0WxmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7NUp0lBUoOkzxQVjJmZlavTpBARO6ncT9nMzAaALDfZ+Q9J3wJ+CPyxvTAiHsstKjMzK0WWpHB08ry4qiyA43s/HDMzK1OXSSEijisiEDMzK1+XZx9J2kfSNyWtTR7fkLRPEcGZmVmxspySugz4A/DXyWMHsDzPoMzMrBxZ+hQOjoiPVc1/RdK6vAIyM7PyZGkpvCppZvuMpGOAV/MLyczMypKlpXAR8L2qfoT/BObnF5KZmZWl06QgaRDw3og4XNJwgIjYUUhkZmZWuK6uaH4LuDiZ3uGEYGbWv2XpU7hH0mclHSBpZPsj98jMzKxwWfoULkieP1VVFsBBvR+OmZmVKUufwrkR8R8FxWNmZiXK0qfw9YJiMTOzkmXpU7hb0sckKfdozMysVFn6FBYCewFtkl4DBEREDM81MjMzK1yWUVL3LiIQMzMrX93DR5LOrZo+psOyi/MMyszMytFZn8LCqul/6rDsAnaDpBGSfiTpl5I2SDoquf7hHkkbk+d9d6cOMzPrvs6SgupM15rvrmuAuyLiEOBwYAOwCFgdEZOB1cm8mZkVqLOkEHWma81nloyh9EHguwAR8UZEvAzMBVYkq60ATutpHWZm1jOddTQfIulxKq2Cg5NpkvnduZr5IGArsFzS4cCjwKXAfhHRChARrZLG1tpY0gJgAcDEiRN3IwwzM+uos6TwvhzrbAI+HRFrJF1DNw4VRcRSYClAc3Nzj1ssZmb2dnWTQkQ8l1OdLUBLRKxJ5n9EJSlsljQ+aSWMB7bkVL+ZmdWR5YrmXhURvwN+K+m9SdFsYD2wij/fvGc+cFvRsZmZDXRZrmjOw6eB6yUNAX4NnE8lQd0o6ULgeeCMkmIzMxuwMiUFSUOBiRHxdG9UGhHrgOYai2b3xuubmVnPdHn4SNIcYB1wVzI/TdKqvAMzM7PiZelTuBKYDrwM6X/5k/ILyczMypIlKbRFxPbcIzEzs9Jl6VN4UtLZQIOkycAlwE/yDcvMzMqQpaXwaeBQ4HXg/wDbgcvyDMrMzMrR1T2aG4CvRMTfAl8sJiQzMytLV/do3gkcWVAsZmZWsix9Cj9PTkG9Cfhje2FE3JxbVGZmVoosSWEksA04vqosACcFM7N+Jss9ms8vIhAzMytfl0lBUiNwIZUzkBrbyyNit27JaWZmfU+WU1K/D4wDTgQeACYAf8gzKDMzK0eWpPAXEfF3wB8jYgVwCnBYvmGZmVkZsiSFN5PnlyVNBfbBYx+ZmfVLWc4+WippX+DvqNwIZxjwpVyjMjOzUmQ5++g7yeQDwEH5hmNmZmXKcvZRzVZBRCzu/XDMzKxMWQ4f/bFquhH4MLAhn3DMzKxMWQ4ffaN6XtLXqfQtmJlZP5Pl7KOO9sR9C2Zm/VKWPoUnqIx1BNAAjAHcn2Bm1g9l6VP4cNV0G7A5ItpyisfMzEqUJSl0HNJiuKR0JiJ+36sRmZlZabIkhceAA4D/BASMAJ5PlgXuXzAz6zeydDTfBcyJiNERMYrK4aSbI+LAiHBCMDPrR7IkhQ9ExB3tMxFxJ3BsfiGZmVlZshw+eknSFcAPqBwuOpfKndjMzKyfydJSmEflNNRbgFuBsUmZmZn1M1muaP49cClAMlrqyxERnW9lZmbvRHVbCpK+JOmQZPpdkv4d2ARslnRCUQGamVlxOjt8dCbwdDI9P1l3LJVO5n/IOS4zMytBZ0nhjarDRCcCN0TEzojYQLYOajMze4fpLCm8LmmqpDHAccDdVcv2zDcsMzMrQ2f/8V8K/IjKmUf/MyJ+AyDpQ8DPC4jNzMwKVjcpRMQa4JAa5XcAd7x9CzMze6fryf0UeoWkBkk/l3R7Mj9S0j2SNibP+5YVm5nZQFVaUqByeKr6tp6LgNURMRlYncybmVmBSkkKkiYApwDfqSqeC6xIplcApxUdl5nZQJfp1FJJRwOTqtePiO/tRr1XA58D9q4q2y8iWpPXbpU0tk4sC4AFABMnTtyNEMzMrKMuWwqSvg98HZgJfCB5NPe0QkkfBrZExKM92T4ilkZEc0Q0jxkzpqdhmJlZDVlaCs3AlF4c7+gY4NTk1NZGKndy+wGV4TPGJ62E8cCWXqrPzMwyytKn8CQwrrcqjIgvRMSEiJgEnAX8e0ScC6yiMpwGyfNtvVWnmZllk6WlMBpYL+kR4PX2wog4tZdjWQLcKOlCKrf7PKOXX9/MzLqQJSlcmVflEXE/cH8yvQ2YnVddZmbWtSz3U3igiEDMzKx8Wc4+miHpZ5JekfSGpJ2SdhQRnJmZFStLR/O3qNx+cyMwFPjvSZmZmfUzmS5ei4hNkhoiYiewXNJPco7LzMxKkCUp/EnSEGCdpH8EWoG98g3LzMzKkOXw0XnJehcDfwQOAD6WZ1BmZlaOLGcfPSdpKDA+Ir5SQExmZlaSLGcfzQHWAXcl89Mkrco7MDMzK16Ww0dXAtOBlwEiYh2VEVPNzKyfyZIU2iJie+6RmJlZ6bKcffSkpLOBBkmTgUsAn5JqZtYPZWkpfBo4lMpgeDcAO4DL8gzKzMzKkeXsoz8BX0weZmbWj9VNCl2dYZTD0NlmZlayzloKRwG/pXLIaA2gQiIyM7PSdJYUxgF/RWUwvLOB/wfcEBFPFRGYmZkVr25Hc0TsjIi7ImI+MAPYBNwv6dOFRWdmZoXqtKNZ0ruAU6i0FiYB1wI35x+WmZmVobOO5hXAVOBO4CsR8WRhUZmZWSk6aymcR2VU1L8ELpHSfmYBERHDc47NzMwKVjcpRESWC9vMzKwf8Q+/mZmlnBTMzCzlpGBmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWarwpCDpAEn3Sdog6SlJlyblIyXdI2lj8rxv0bGZmQ10ZbQU2oD/ERHvo3Kbz09JmgIsAlZHxGRgdTJvZmYFKjwpRERrRDyWTP8B2ADsD8wFViSrrQBOKzo2M7OBrtQ+BUmTgCOANcB+EdEKlcQBjK2zzQJJayWt3bp1a1GhmpkNCKUlBUnDgH8DLouIHVm3i4ilEdEcEc1jxozJL0AzswGolKQgaTCVhHB9RNycFG+WND5ZPh7YUkZsZmYDWRlnHwn4LrAhIr5ZtWgVMD+Zng/cVnRsZmYD3R4l1HkMcB7whKR1SdnlwBLgRkkXAs8DZ5QQm5nZgFZ4UoiIhwDVWTy7yFjMzGxXvqLZzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmKScFMzNLOSmYmVnKScHMzFJOCmZmlnJSMDOzlJOCmZmlnBTMzCzlpGBmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7NUn0sKkk6S9LSkTZIWlR2PmdlAskfZAVST1AD8M/BXQAvwM0mrImJ9b9bzVtub7HxtB2gQAjSoASSE0KBBlWlVytAgQMm0ejMMM7M+p08lBWA6sCkifg0gaSUwF+jVpLDp8Z/wl6tO7dG2b4V4CxHsmiDibWt2XK6q6c51fO2uqMvad9+f64hdXl/p/K7lg1T/Xb4VlbXb32fs8irtZe3ltd9J7ffc1Z59u672dXc/i6x6Emt35BV3Z/J+T7arp0YcR/Nnbuz11+1rSWF/4LdV8y3Af6leQdICYEEy+4qkp3ejvtHAS7uxfV4cV/c4ru5xXN3TR+O6aTQL1dO43lNvQV9LCrX+vdnl34+IWAos7ZXKpLUR0dwbr9WbHFf3OK7ucVzdM9Di6msdzS3AAVXzE4AXS4rFzGzA6WtJ4WfAZEkHShoCnAWsKjkmM7MBo08dPoqINkkXAz8GGoBlEfFUjlX2ymGoHDiu7nFc3eO4umdAxaUInzFgZmYVfe3wkZmZlchJwczMUv0+KXQ1bIYqrk2WPy6pqYCYDpB0n6QNkp6SdGmNdWZJ2i5pXfL4Ut5xJfU+K+mJpM61NZaXsb/eW7Uf1knaIemyDusUtr8kLZO0RdKTVWUjJd0jaWPyvG+dbXMbxqVOXFdJ+mXyWd0iaUSdbTv93HOI60pJL1R9Xh+qs23R++uHVTE9K2ldnW1z2V/1fhsK/X5FRL99UOms/hVwEDAE+AUwpcM6HwLupHKNxAxgTQFxjQeakum9gWdqxDULuL2EffYsMLqT5YXvrxqf6e+A95S1v4APAk3Ak1Vl/wgsSqYXAV/ryfcxh7j+G7BHMv21WnFl+dxziOtK4LMZPutC91eH5d8AvlTk/qr321Dk96u/txTSYTMi4g2gfdiManOB70XFw8AISePzDCoiWiPisWT6D8AGKldzvxMUvr86mA38KiKeK7DOXUTEg8DvOxTPBVYk0yuA02psmuX72KtxRcTdEdGWzD5M5dqfQtXZX1kUvr/aSRLw18ANvVVfxpjq/TYU9v3q70mh1rAZHX98s6yTG0mTgCOANTUWHyXpF5LulHRoQSEFcLekR1UZUqSjUvcXlWtX6v2hlrG/2u0XEa1Q+cMGxtZYp+x9dwGVVl4tXX3uebg4Oay1rM7hkDL3138FNkfExjrLc99fHX4bCvt+9fek0OWwGRnXyYWkYcC/AZdFxI4Oix+jcojkcOCfgFuLiAk4JiKagJOBT0n6YIflZe6vIcCpwE01Fpe1v7qjzH33RaANuL7OKl197r3tfwMHA9OAViqHajoqbX8B8+i8lZDr/urit6HuZjXKur2/+ntSyDJsRilDa0gaTOVDvz4ibu64PCJ2RMQryfQdwGBJo/OOKyJeTJ63ALdQaZJWK3MokpOBxyJic8cFZe2vKpvbD6Mlz1tqrFPWd20+8GHgnEgOPneU4XPvVRGxOSJ2RsRbwL/Uqa+s/bUH8FHgh/XWyXN/1fltKOz71d+TQpZhM1YBH0/OqpkBbG9vpuUlOV75XWBDRHyzzjrjkvWQNJ3KZ7Ut57j2krR3+zSVTsonO6xW+P6qUve/tzL2VwergPnJ9HzgthrrFD6Mi6STgM8Dp0bEn+qsk+Vz7+24qvuhPlKnvrKGvTkB+GVEtNRamOf+6uS3objvV2/3nve1B5WzZZ6h0iv/xaTsIuCiZFpUbuzzK+AJoLmAmGZSadY9DqxLHh/qENfFwFNUziB4GDi6gLgOSur7RVJ3n9hfSb17UvmR36eqrJT9RSUxtQJvUvnv7EJgFLAa2Jg8j0zWfTdwR2ffx5zj2kTlOHP79+y6jnHV+9xzjuv7yffncSo/XOP7wv5Kyv+1/XtVtW4h+6uT34bCvl8e5sLMzFL9/fCRmZl1g5OCmZmlnBTMzCzlpGBmZiknBTMzSzkpmHVB0k7tOkprr43WKWlS9SidZmXrU7fjNOujXo2IaWUHYVYEtxTMeigZU/9rkh5JHn+RlL9H0upksLfVkiYm5fupck+DXySPo5OXapD0L8n4+XdLGlram7IBz0nBrGtDOxw+OrNq2Y6ImA58C7g6KfsWleHF309lALprk/JrgQeiMmhfE5WrYQEmA/8cEYcCLwMfy/n9mNXlK5rNuiDplYgYVqP8WeD4iPh1MojZ7yJilKSXqAzb8GZS3hoRoyVtBSZExOtVrzEJuCciJifznwcGR8RX839nZm/nloLZ7ok60/XWqeX1qumduK/PSuSkYLZ7zqx6/mky/RMqI1QCnAM8lEyvBv4GQFKDpOFFBWmWlf8jMevaUO16A/e7IqL9tNR3SVpD5R+seUnZJcAySX8LbAXOT8ovBZZKupBKi+BvqIzSadZnuE/BrIeSPoXmiHip7FjMeosPH5mZWcotBTMzS7mlYGZmKScFMzNLOSmYmVnKScHMzFJOCmZmlvr/TVb4o1zHli4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_history(history):\n",
    "  hist = pd.DataFrame(history.history)\n",
    "  hist['epoch'] = history.epoch\n",
    "  \n",
    "  plt.figure()\n",
    "  plt.xlabel('Epoch')\n",
    "  plt.ylabel('Mean Square Error')\n",
    "  plt.plot(hist['epoch'], hist['mean_squared_error'],\n",
    "           label='Train Error')\n",
    "  plt.plot(hist['epoch'], hist['val_mean_squared_error'],\n",
    "           label = 'Val Error')\n",
    "  plt.ylim([0,100])\n",
    "  plt.legend()\n",
    "  plt.show()\n",
    "\n",
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load best weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 22, 22, 32)        3904      \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 20, 20, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 10, 10, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 8, 8, 32)          9248      \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 6, 6, 32)          9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 3, 3, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 288)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 32)                9248      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 64)                2112      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 43,073\n",
      "Trainable params: 43,073\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "[0.036803836992974084, 0.008001861973785078, 0.036803836992974084, 0.9902]\n"
     ]
    }
   ],
   "source": [
    "best_model = keras.models.load_model('model/CNN_test.h5')\n",
    "best_model.summary()\n",
    "results = best_model.evaluate(testimages, testlabels, verbose = 0)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate the performance of our DNN! Plot predictions vs. true values (the line is predictions vs. predicitons)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxOdfvA8c81M/YtSSpL1kdmhrEvqSg9lhKRJ6mnRzEhSqV+og3thbKkkPFIRYvKUpGlRHYSMRIPYpCQbWQyy/X74xzjNsa4jbnnzD1zvV+v+zVn+d7nXPeZmXPd3+/3nO8RVcUYY4w5lxCvAzDGGJOzWaIwxhiTIUsUxhhjMmSJwhhjTIYsURhjjMmQJQpjjDEZskRhchQRGSsiz3odh9dE5B4Rmet1HMYAiN1HYYy3RKQisB3Ip6pJ3kZjzNmsRmEumIiEeR3DhRCHZ3/rIhLq1b4DxetjarKX/aKNX0Rkh4g8KSLrgeMiEiYiV4nIZyKyX0S2i0hfn/KhIvKUiPxPRI6JyBoRKe+uu0ZE5onInyKyWUTu9HnfJBF50Z3eJCJtfdaFicgBEanrzjcWkaUiclhE1olIc5+yC0XkJRFZAvwFVE7nM9Vwyx0WkY0i0i5NHGPdOI+JyPcicrXP+vN9hndE5GsROQ7cKCK3ishaETkqIrtEZLBPKIvcn4dFJF5EmojIfSLyg882VUR6icgWETkkImNERHyO9XD32GwXkYfc8ukmdBEpLyKfu7+3gyLylrt8sIh84FOuou920jmmT4nI6jTbfkxEZrrTBURkmIjsFJF97vEslF5MJodTVXvZ67wvYAfwE1AeKITzJWMN8ByQH+dEvA1o5Zb/P+BnoDogQBRQCigC7ALuB8KAusABIMJ93yTgRXf6OeBDnxhuBX5xp8sCB4Fb3Fj+6c6XdtcvBHYCEe5+8qX5PPmArcBTbvw3AceA6j5xHANuAAoAI4Ef3HX+fIYjQFM3toJAc6CmO18L2Afc7pavCCgQ5hPffaf2584r8CVwCVAB2A+0dtf1AmKBckBJYH7a7flsJxRYB7zpfo6CwHXuusHABz5lz4grnWNawj1G1Xzeswq4y50eAcwELgWKAbOAV7z+W7ZXJv7/vQ7AXsHxwkkU3XzmGwE705QZCPzXnd4MtE9nO52BxWmWjQMGudOTOJ0oqronosLu/IfAc+70k8D7abbzDdDVnV4IPJ/B57ke+B0I8Vk2FRjsE8dHPuuKAsk4idKfzzD5PMdzBPCmO+1vorjOZ/4TYIA7/S3Q02fdzRkkiiY4SSa9df4kiufTvOcDn99JtVO/L5wvB8eBKmn2vd3rv2V7XfgrqNqajed2+UxfDVwlIod9loUCi93p8sD/0tnG1UCjNO8LA95PW1BVt4rIJuA2EZkFtAPq+GznXyJym89b8gHfnSPetK4Cdqlqis+y33BqKme9X1XjReRP933+fIYz9i0ijYBXgUicGkwB4NMM4kvP7z7Tf+Ekr9TPcq59p1Ee+E0z32medttTgOHA88DdwHRV/UtELsdJGGvcFjJwkkeu66/JCyxRmAvhe4ncLpxvh9XOUXYXUAXYkM7y71X1n37ucyrQBafJJlZVt/ps531VfcDPeNPaA5QXkRCfZFEB+NWnTPlTEyJSFKcJZY+fnyHtvqcAbwFtVDVBREYAl/kRpz/24jQ7nRV3OnYBFUQkLJ1kcRzn5H7KFem8P22sc4HLRKQ2zu/pMXf5AeAETnPc7vPEb3I468w2mbUSOOp2cBdyO1QjRaSBu34C8IKIVHOvkKklIqVw2tn/ISL3ikg+99VARGqcYz8fAS2BB3FOtqd8gFPTaOXuu6CINBeRculu5WwrcE6M/d0YmgO3ufs75RYRuU5E8gMvACtUdVcmPgM4bfR/ukmiIc6371P2Aymk0+Hup0+AR0SkrIhcgtMsdy4rcRLLqyJSxD1uTd11PwE3iEgFESmB05SYITfZTAOG4iTSee7yFOBd4E23doEbX6vMfUTjJUsUJlNUNRnnxFob5x6AAzjJoYRb5A2cE9hc4CgQAxRS1WM4J/67cL6d/w68htMUk95+9gLLgGuBj32W7wLa43RG78f5pvx/+Pk3raoncZqy2rixvw38R1V/8Sk2BRgE/AnUA+5x33tBn8HVG3heRI7hdNJ/4hPLX8BLwBL3CqzG/nwGH+/iHOf1wFrgayAJp08l7ec+9XuritMxHYfT54KqzsM5xutxLlT40s/9T8HpF/k0TS3lSZwLBpaLyFGcTvbqF/jZTA5gN9wZkw4RmQTEqeozXsdyoUSkDTBWVa8+b2Fj/GA1CmOCnNv0d4s495mUxakFfeF1XCb3CFiiEJGJIvKHiKTtzDy1XkRklIhsFZH14t5EZYy5YAIMAQ7hND1twmneMiZLBKzpSURuAOJxriePTGf9LcDDODdMNQJGqmqjgARjjDEm0wJWo1DVRTidgOfSHieJqKouBy4RkSsDFY8xxpjM8fI+irKcefNOnLtsb9qCItID6AFQpEiRetdccw3HjsEff0B8PIhAYmK2xGyMMUHkOM6gCn8DekBVS2dmK14mCklnWbrtYKo6HhgPUK9efa1XbzXjx59ZJiQEUlIgPByOHYMyZSA5GerUgeLFYdcuiIx0kkpIyOmXv/P79kGFCll9CLwh6R35IGWfJWeyz+KthIR4pk17hrlzR6GqXHHFP/j9919/y+z2vEwUcZx5B2k5nGvSM/T776QmicaN4dlnoUoVuOoqKFYsIHEaY0zQmDdvHj169GDHjh2EhobSv39/nnvuOQoVyvzAvV4mipnAQyLyEU5n9hH35qoM7XFTSefO8NFHGZc1xpi8ZPXq1bRs2RKA2rVrExMTQ926F39BacAShYhMxRla+TIRicO5tjsfgKqOxbl79BacOzf/whmy2W8vv5yV0RpjTPCrX78+d999N5GRkTzxxBPky5cvS7YbdHdmi9RXWE1KSnC2HRpjTFbZt28fjz76KAMGDCAqKgpwHh0h6ZwcRWSNqtbPzH6CdvRYSxLGmLxKVXn//fd59NFHOXToEHv37mXhwoUA6SaJixWUiSILmtyMMSYo/fbbb/Ts2ZNvvvkGgFatWjFu3LiA7jMox3qqWtXrCIwxJnulpKQwZswYIiMj+eabbyhZsiSTJk1i9uzZXH11YMd/DMoaxbFjXkdgjDHZa+/evQwYMID4+Hg6derE6NGjueKK9J4tlfWCMlE0bOh1BMYYE3hJSUmEhIQQEhJC2bJleeuttyhWrBgdO3bM1jiCsulp/36vIzDGmMBau3YtDRs2ZOzYsanLunbtmu1JAoI0UVxzjdcRGGNMYCQkJPDUU0/RoEED1q5dy9ixY0lOPuthhdkqKBNFWFA2mBljTMZ++OEHateuzSuvvEJKSgp9+/Zl6dKlhIaGehpXUJ5yQ4IyvRljTPpOnDhB//79GTNmDKrKNddcQ0xMDNdee63XoQFBWqPwOLkaY0yWypcvX2rN4emnn2bt2rU5JkmA1SiMMcYTf/75J8nJyZQuXZqwsDAmT55MYmIitWvX9jq0swTlKddqFMaYYPbZZ58RHh7Ogw8+mLosIiIiRyYJCNJEYTUKY0ww2rt3L3fccQedOnVi3759/PHHH8THx3sd1nkF5SnXahTGmGCiqvz3v/8lPDyczz//nKJFi/L222+zcOFCihYt6nV452V9FMYYE0DJycm0bduWOXPmANCmTRvGjh1LhSB6tnJQnnITE72OwBhj/BMaGkqNGjUoVaoU77//Pl999VVQJQkI0gcXTZmymi5dvI7EGGPSt2nTJv7880+aNm0KwPHjxzl+/DiXX365ZzFdzIOLgrJGUayY1xEYY8zZEhMTeemll6hduzZdunThmDvUdZEiRTxNEhcrKPsojDEmp1mzZg3du3dn3bp1gPNAoWBrsTmXoKxRGGNMTnHixAkGDBhAo0aNWLduHZUqVWL+/Pm8++67FC9e3OvwsoTVKIwx5iK0bduWb7/9FhHhscce44UXXqBIkSJeh5WlLFEYY8xF6NevH7///jsxMTE0btzY63ACwpqejDHmAsyePZuhQ4emzt96662sW7cu1yYJsBqFMcb45cCBAzz22GN88MEHhISE0LJlS6KiogAIy+UPyQnKGoWI1xEYY/IKVeWTTz4hPDycDz74gIIFC/Laa68RERHhdWjZJnenQWOMuQh79uyhd+/ezJgxA4BmzZoxYcIEqlat6nFk2SsoaxTGGJMd+vfvz4wZMyhWrBjjxo3j22+/zXNJAqxGYYwxZ1BVxG3ffv3110lOTmbo0KGUK1fO48i8YzUKY4zBGeX1zTffpEWLFiQnJwNw1VVXMXXq1DydJMBqFMYYw8aNG+nevTsrVqwAnEtg27Zt63FUOYfVKIwxedbJkyd5/vnnqVOnDitWrKBs2bLMnDnTkkQaQVmjsMtjjTEXa9WqVXTr1o0NGzYA0LNnT1577TVKlCjhcWQ5T1AmCmOMuVjLli1jw4YNVKlShXfffZcbb7zR65ByLEsUxpg8Y//+/ZQuXRqAPn36oKo88MADFC5c2OPIcjbrozDG5HpHjhyhV69eVK1alV27dgHOI0ofeeQRSxJ+CGiiEJHWIrJZRLaKyIB01pcQkVkisk5ENorI/YGMxxiT93z11VdEREQwbtw4Tpw4wdKlS70OKegELFGISCgwBmgDhANdRCQ8TbE+QKyqRgHNgeEikj9QMRlj8o79+/dzzz330LZtW3bv3k2jRo1Yu3YtnTt39jq0oBPIGkVDYKuqblPVk8BHQPs0ZRQoJs5tkEWBP4GkAMZkjMkD5syZQ3h4OFOmTKFQoUK88cYbLFmyJE8N5JeVAtmZXRbY5TMfBzRKU+YtYCawBygGdFbVlLQbEpEeQA9nrl4AQjXG5CZlypTh0KFD3HTTTbz77rtUrlzZ65CCWiBrFOnd7ZD2SeOtgJ+Aq4DawFsictZDZlV1vKrWV9X6YPdRGGPOlJKSwty5c1Pn69Spw/Lly5k/f74liSwQyEQRB5T3mS+HU3PwdT/wuTq2AtuBawIYkzEml9m6dSstWrSgVatWTJ8+PXV5/fr1Uwf3MxcnkIliFVBNRCq5HdR34TQz+doJtAAQkTJAdWBbAGMyxuQSycnJDB8+nFq1arFw4UJKly5NSIhd8R8IAeujUNUkEXkI+AYIBSaq6kYR6eWuHwu8AEwSkZ9xmqqeVNUDgYrJGJM7bNiwgW7durFq1SoA/v3vfzNixAhKlSrlcWS5U0DvzFbVr4Gv0ywb6zO9B2gZyBiMMbnLnDlzaNeuHYmJiZQrV45x48Zxyy23eB1WrmZDeBhjgsp1111HuXLlaN26Na+++irFi591/YvJYpYojDE52vHjxxk2bBiPP/44RYsWpWjRoqxbt45ixYp5HVqeEZSJwi5kMCZv+Pbbb3nggQfYtm0bBw8eZNSoUQCWJLKZXSJgjMlxDh8+zAMPPECLFi3Ytm0bUVFRdO3a1euw8ixLFMaYHGXmzJlEREQwYcIE8ufPz4svvsiqVauoV89GZfBKUDY9GWNyp1WrVtG+vTMkXJMmTYiJiaFGjRoeR2UsURhjcowGDRrQrVs3oqKi6NOnD6GhoV6HZLCmJ2OMh3bt2kX79u358ccfU5fFxMTQt29fSxI5iNUojDHZLiUlhXHjxvHkk09y7Ngxjh49ynfffed1WOYcLFEYY7LVr7/+ygMPPMCiRYsA6NChA2PGjPE4KpORoGx6svsojAk+SUlJvP7660RFRbFo0SLKlCnDtGnT+Pzzz7nyyiu9Ds9kICgThTEm+Ozbt48XX3yRhIQEunbtSmxsLHfccYfXYRk/WNOTMSZg/v77b8LCwggNDaVs2bKMGzeOkiVL0rp1a69DMxfAahTGmIBYtmwZderU4a233kpd1qVLF0sSQcgShTEmS8XHx/Poo4/StGlTNm3axOTJk0lJSfE6LHMRLFEYY7LMvHnzqFmzJiNHjiQkJISBAweyZMkSe/JckLM+CmPMRTt+/Dh9+/Zl4sSJANSuXZuJEydSp04djyMzWSEo07xdHmtMzlKwYEE2bNhAgQIFeOWVV1i5cqUliVzkvDUKERHgHqCyqj4vIhWAK1R1ZcCjM8bkWL///jsAV1xxBaGhoUyePBmA6tWrexmWCQB/ahRvA02ALu78McBuozQmj1JV3nvvPcLDw+nZsyeqCjgJwpJE7uRPH0UjVa0rImsBVPWQiOQPcFzGmBzot99+o2fPnnzzzTeAc5/EX3/9RZEiRTyOzASSPzWKRBEJBRRAREoDdq2bMXlISkoKY8aMITIykm+++YaSJUvy3nvvMXv2bEsSeYA/NYpRwBfA5SLyEtAJeDagURljcozk5GRuvvlmFi5cCECnTp146623KFOmjLeBmWxz3kShqh+KyBqgBSDA7aq6KeCRGWNyhNDQUBo2bMgvv/zCmDFj6Nixo9chmWwmpzqizllA5H1Vvfd8y7KLSH395pvVtGzpxd6NyRvWrl3L4cOHufHGGwE4ceIECQkJlCxZ0uPITGaJyBpVrZ+Z9/rTRxGRZmehgKdPObf7KIwJjISEBJ566ikaNGjAv//9b44cOQJAoUKFLEnkYedsehKRgcBTQCEROYrT7ARwEhifDbEZY7LRkiVL6N69O5s3b0ZE+Ne//kVYmA3eYDKoUajqK6paDBiqqsVVtZj7KqWqA7MxRmNMAB07doyHH36Y66+/ns2bN1OjRg2WLFnCiBEj7IomA/jXmT1QREoC1YCCPssXBTIwY0z2aNu2LYsWLSIsLIwBAwbwzDPPUKBAAa/DMjmIP0N4RAOPAOWAn4DGwDLgpsCGZozJDgMHDuT48ePExMQQFRXldTgmB/KnM/sRoAHwm6reCNQB9gc0KmNMwEybNo0XXnghdb5169asXLnSkoQ5J396qhJUNUFEEJECqvqLiNiALsYEmb179/LQQw/x+eefIyK0a9cuNTnY8yJMRvxJFHEicgkwHZgnIoeAPYENyxiTVVSVSZMm0a9fPw4fPkyxYsV4/fXXqVmzptehmSDhT2d2B3dysIh8B5QA5gQ0qvOw+yiM8c/27dvp0aMH8+fPB6BNmzaMGzeO8uXLexyZCSYZ1jdFJERENpyaV9XvVXWmqp70Z+Mi0lpENovIVhEZcI4yzUXkJxHZKCLfX1j4xpiMPPvss8yfP59SpUrxwQcf8NVXX1mSMBcswxqFqqaIyDoRqaCqOy9kw+4d3GOAfwJxwCoRmamqsT5lLsF53kVrVd0pIpdf+EcwxvhKTk4mNDQUgGHDhqU+de7yy+3fy2SOPz1YVwIbRWSBiMw89fLjfQ2Braq6za2BfAS0T1PmbuDzU0lIVf+4kOCNMaclJiby4osvcsMNN5CUlAQ4T5+LiYmxJGEuij+d2UMyue2ywC6f+TigUZoy/wDyichCoBgwUlUnp92QiPQAejhzng4zZUyOtGbNGrp168b69esBmDdvHm3atPE4KpNb+NOZndl+g/S6nNMOVRuGc+ZvARQClonIclX9NU0M43HHlxKpn/Fwt8bkISdOnGDw4MEMGzaMlJQUKleuzLvvvstNN9n9sCbrBHLErzjAt9esHGdfVhsHHFDV48BxEVkERAG/YozJ0OLFi+nevTtbtmwhJCSEfv368fzzz9v4TCbLBfIum1VANRGp5D5j+y4gbd/GDOB6EQkTkcI4TVPnfSiSXR5rDPz8889s2bKFiIgIli5dyvDhwy1JmIDwq0YhIoWACqq62d8Nq2qSiDwEfAOEAhNVdaOI9HLXj1XVTSIyB1iP8xzuCaq64dxbNSZvi4uLo1y5cgD06tWLfPny0bVrV/Lnz+9xZCY38+cJd7cBw4D8qlpJRGoDz6tqu+wI8Ox46uv8+atp0cKLvRvjjQMHDvDoo48yffp0Nm7cyNVXX+11SCbIBPoJd4NxLnU9DKCqPwEVM7MzY8yFUVU+/vhjwsPD+fDDD0lJSeHHH3/0OiyTx/iTKJJU9UjAIzHGnGHPnj106NCBu+66i/3799O8eXPWr19Phw4dzv9mY7KQP4lig4jcDYSKSDURGQ0sDXBcxuRpM2fOJDw8nBkzZlC8eHHGjRvHggULqFq1qtehmTzIn0TxMBAB/A1MAY4AjwYyKGPyuooVK3L8+HHatm3Lxo0b6dGjhw0Fbjzjz1VP1VX1aeDpQAdjTF6VnJzMrFmzaN++PSJCrVq1WLt2LREREYhdD2485s9XlDdE5BcReUFEIgIekR/s/8bkJhs3bqRp06Z06NCBadOmpS6PjIy0JGFyhPMmCvfxp81xHn86XkR+FpFnAh2YMbndyZMnef7556lTpw4rVqygbNmyFC9e3OuwjDmLX42eqvq7qo4CegE/Ac8FNCpjcrlVq1ZRv359Bg0aRGJiIj179mTjxo20atXK69CMOct5+yhEpAbQGegEHMQZLvzxAMdlTK715Zdf0r59e1JSUqhSpQoTJkygefPmXodlzDn505n9X2Aq0FJV7VnZxlykm266iapVq9KuXTuGDBlC4cKFvQ7JmAz5M8x44+wIxJjc6siRI7z88ss8/fTTFC9enMKFC7Nu3ToKFizodWjG+OWciUJEPlHVO0XkZ858joQAqqq1Ah6dMUHuyy+/pFevXuzevZv4+HjGjBkDYEnCBJWMahSPuD/bZkcgF8KuGDQ53f79+3nkkUeYOnUqAI0aNaJ3794eR2VM5pzzqidV3etO9lbV33xfgP3FG5MOVWXKlCnUqFGDqVOnUrhwYd58802WLFlCRESOuA3JmAvmz+Wx/0xnmT2M15h0rFy5knvuuYeDBw/SokULfv75Zx599FFCQ0O9Ds2YTMuoj+JBnJpDZRFZ77OqGLAk0IEZE4xONTHVrVuXbt262Z3VJlfIqI9iCjAbeAUY4LP8mKr+GdCojAkSW7dupVevXrz88ss0bNgQILXD2pjcIqOmJ1XVHUAf4JjPCxG5NPChGZNzJSUlMWzYMGrWrMmCBQsYOHCg1yEZEzDnq1G0BdbgXB7rW4dWoHIA4zImx1q/fj3du3dn9erVANx77728+eabHkdlTOCcM1Goalv3Z6XsC8eYnOvvv//m5Zdf5uWXXyYpKYny5cszbtw42rSxaztM7nbeq55EpKmIFHGn/y0ib4hIhcCHllFMXu7d5FV//vknI0eOJCkpiT59+rBx40ZLEiZP8Gesp3eAKBGJAvoDMcD7QLNABmZMTnD8+HEKFChAWFgYV155JTExMVx++eVcf/31XodmTLbx5z6KJFVVoD0wUlVH4lwia0yutmDBAmrWrMnIkSNTl91xxx2WJEye40+iOCYiA4F7ga9EJBTIF9iwjPHO4cOHiY6O5uabb2b79u189tlnpKSkeB2WMZ7xJ1F0Bv4Guqnq70BZYGhAozLGIzNmzCA8PJyYmBjy58/PSy+9xPfff09IiF/P+DImV/JnmPHfReRDoIGItAVWqurkwIdmTPaJj48nOjqajz/+GIAmTZoQExNDjRo1PI7MGO/5c9XTncBK4F/AncAKEekU6MCMyU6FCxcmLi6OIkWKMGrUKBYvXmxJwhiXP1c9PQ00UNU/AESkNDAfmBbIwDJil8earLBr1y5CQkIoW7YsISEhvPfee4SGhlKxYkWvQzMmR/Gn4TXkVJJwHfTzfcbkSCkpKbzzzjuEh4fzwAMP4FzUB1WqVLEkYUw6/KlRzBGRb3Cemw1O5/bXgQvJmMD59ddfiY6OZvHixQAUKlSIEydO2HOrjcnAeWsGqvp/wDigFhAFjFfVJwMdmDFZKSkpiddff52oqCgWL15MmTJlmDZtGp999pklCWPOw58aBcBSIBlIAVYFLhxjsl5SUhLXXXcdK1asAKBr16688cYbXHqpDYJsjD/8ueopGueqpw5AJ2C5iHQLdGDGZJWwsDBatGhBhQoVmDNnDpMmTbIkYcwFkFMdeecsILIZuFZVD7rzpYClqlo9G+JLJ576unDhaprZSFMmA0uXLiU+Pp6WLVsCkJCQQGJiIsWK2egzJm8SkTWqWj8z7/Xn6qU43AcWuY4BuzKzM2MCLT4+nkceeYTrrruOrl27cujQIQAKFixoScKYTPInUezGuclusIgMApYDW0Wkn4j0y+iNItJaRDaLyFYRGZBBuQYikuzvjXx2H4VJz9y5c4mMjGTUqFGEhITQrVs3ChUq5HVYxgQ9fzqz/+e+Tpnh/szw65k7eOAY4J84tZJVIjJTVWPTKfca8I2/QRvj69ChQ/Tr149JkyYBUKdOHSZOnEjt2rW9DcyYXMKfsZ6GZHLbDYGtqroNQEQ+whmqPDZNuYeBz4AGmdyPyePatm3L0qVLKVCgAIMHD+bxxx8nXz4b4NiYrBLIO6zLcmZfRpy7LJWIlMW5mmpsRhsSkR4islpEVmd5lCboDRkyhOuvv55169YxYMAASxLGZLFAJor0ehLSXmI1AnhSVZMz2pCqjlfV+pntsTe5h6ry3nvv8cwzz6Quu/nmm/n++++pXt2TC/GMyfX8veEuM+KA8j7z5YA9acrUBz4Sp3f6MuAWEUlS1ekBjMsEqR07dtCzZ0/mzp0LwJ133kmtWrUAELvCwZiA8eeGu3+IyAIR2eDO1xKRZ873Ppw7uKuJSCURyQ/cBcz0LaCqlVS1oqpWxBmNtrclCZNWSkoKo0ePJjIykrlz53LppZcyefJkatas6XVoxuQJ/jQ9vQsMBBIBVHU9zkk/Q6qaBDyEczXTJuATVd0oIr1EpFfmQ7bLY/OSX375hRtuuIG+ffty/Phx7rzzTmJjY7n33nutFmFMNvGn6amwqq5M80+Z5M/GVfVr0ow0q6rpdlyr6n3+bNPkLS+99BJLlizhiiuu4J133uH222/3OiRj8hx/EsUBEamC2xHt3hS3N6BRmTwtMTEx9cqlN954g5IlSzJkyBBKlizpcWTG5E3+ND31wRlm/BoR2Q08CjwY0KhMnnTixAkGDhxIkyZNSExMBKB06dKMGjXKkoQxHvLnhrttwM0iUgTnaXfHzvceYy7UDz/8QPfu3fn1118REb777rvUAf2MMd46b6IQkefSzAOgqs8HKCaThxw7doyBAwcyZswYAGrUqPenDhoAABrySURBVEFMTAxNmjTxODJjzCn+9FEc95kuCLTFuYrJmIsyb948oqOj2blzJ2FhYQwcOJCnn36aAgUKeB2aMcaHP01Pw33nRWQYae6HMCYzduzYwc6dO6lXrx4xMTFERUV5HZIxJh2ZuTO7MFA5qwO5EHb5fHBSVbZt20aVKlUAiI6OpnDhwnTu3JmwsEAOEmCMuRj+3Jn9s4isd18bgc3AyMCHZnKTvXv3cscdd1CzZk22bdsGOP1d99xzjyUJY3I4f/5D2/pMJwH73LuujTkvVWXSpEn069ePw4cPU6xYMWJjY6lc2dNKqTHmAmSYKEQkBPhKVSOzKR6Ti2zfvp0ePXowf/58AG699VbGjh1LuXLlPI7MGHMhMmx6UtUUYJ2IVMimeEwuMW3aNCIjI5k/fz6lSpXiww8/ZNasWZYkjAlC/jQ9XQlsFJGV+Fwqq6rtAhaVCXrh4eEkJSVx1113MXLkSC6//HKvQzLGZJI/iSKzj0I1eUhiYiKffvopXbp0QUQIDw9n48aNVK1a1evQjDEXyZ9EcYuqPum7QEReA74PTEgm2KxevZru3buzfv16RIQuXboAWJIwJpfwZ1DAf6azrE1WB3Ih7D6KnOHEiRP079+fRo0asX79eipXrsxVV13ldVjGmCx2zhqFiDwI9AYqi8h6n1XFgCWBDszkbN9//z3R0dFs3bqVkJAQ+vXrxwsvvEDhwoW9Ds0Yk8UyanqaAswGXgEG+Cw/pqp/BjQqk6NNnz6dDh06ABAREUFMTAyNGjXyOCpjTKCcM1Go6hHgCNAl+8IxwaBVq1ZERkbSqVMnBg4cSP78+b0OyRgTQP70UZg87sCBAzz00EMcPnwYgEKFCvHjjz8yaNAgSxLG5AE2yI45J1Xl448/5uGHH+bAgQMkJyfzzjvvAKQ+qtQYk/tZojDp2r17N71792bmTGdE+RtvvJEnnnjC46iMMV4IyqYnuzw2cFSVd999l/DwcGbOnEnx4sUZP348CxYsSB0e3BiTt1iNwpxhxYoV9OjRA4DbbruNd955h7Jly3oclTHGS5YoDKqa+iz0xo0b069fPxo0aEDnzp1Tlxtj8q6gbHoyWWfDhg1ce+21LFu2LHXZ8OHDueuuuyxJGGMASxR51smTJxkyZAh169Zl+fLlDBo0yOuQjDE5lDU95UErV66ke/fubNiwAYAHH3yQV1991eOojDE5ldUo8pC//vqLJ554giZNmrBhwwaqVq3KwoULefvttylevLjX4RljcihLFHnIsWPHmDhxIgD9+/dn/fr1NGvWzOOojDE5XVA2PVkfq/+OHDlC4cKFyZcvH2XKlOG9997jyiuvpH79+l6HZowJElajyMVmzZpFeHg4w4cPT1122223WZIwxlwQSxS50P79+7n77rtp164de/bsYc6cOaSkpHgdljEmSFmiyEVUlSlTplCjRg2mTp1K4cKFefPNN1mwYAEhIfarNsZkTlD2UZizHT16lLvvvpuvvvoKgBYtWjB+/HgqV67scWTGmGAX0K+ZItJaRDaLyFYRGZDO+ntEZL37WioiUYGMJzcrWrQox44do0SJEsTExDBv3jxLEsaYLBGwGoWIhAJjgH8CccAqEZmpqrE+xbYDzVT1kIi0AcYD9kxNP23ZsoUCBQpQoUIFQkJCmDx5Mvny5eOqq67yOjRjTC4SyBpFQ2Crqm5T1ZPAR0B73wKqulRVD7mzy4Fy/mw4r18em5SUxNChQ6lVqxbR0dGoKgBXX321JQljTJYLZKIoC+zymY9zl51Ld2B2eitEpIeIrBaR1VkYX1Bav349TZo0oX///iQkJHDFFVeQkJDgdVjGmFwskIkive/9mm5BkRtxEsWT6a1X1fGqWl9V8+wNAH///TfPPfcc9erVY/Xq1ZQvX56vv/6ayZMnU6hQIa/DM8bkYoG86ikOKO8zXw7Yk7aQiNQCJgBtVPVgAOMJWklJSTRu3JiffvoJgD59+vDKK69QrFgxjyMzxuQFgUwUq4BqIlIJ2A3cBdztW0BEKgCfA/eq6q8BjCWohYWF0b59e/766y8mTJjA9ddf73VIxpg8RE51hAZk4yK3ACOAUGCiqr4kIr0AVHWsiEwA7gB+c9+SdL7mJZH6umzZaho3DljYOcKCBQtISEjg1ltvBZznRyQnJ1szkzEmU0RkTWab7wOaKAIhtyeKw4cP8/jjjzNx4kRKly7Npk2bKFWqlNdhGWOC3MUkCrszOweZPn06vXv3Zu/eveTPn59HH33UnhNhslViYiJxcXF2JV0QK1iwIOXKlSNfvnxZts2gTBS57T6Kffv28fDDD/Ppp58CcO211zJhwgRq1KjhcWQmr4mLi6NYsWJUrFjRnpkehFSVgwcPEhcXR6VKlbJsuzZSnMdUlfbt2/Ppp59SpEgRRo0axeLFiy1JGE8kJCRQqlQpSxJBSkQoVapUltcILVF4TER47bXXaNWqFRs2bODhhx+2kV6NpyxJBLdA/P6CsukpmKWkpDB27Fi2b9/O0KFDAWjWrJk9ktQYk2PZV9dstHnzZpo1a0afPn0YNmwYP//8s9chGWPS+OWXX2jSpAkFChRg2LBh5yy3fft2GjVqRLVq1ejcuTMnT54EnObkvn37UrVqVWrVqsWPP/6Y+p45c+ZQvXp1qlatyquvvhrwz5JVLFFkg6SkJF599VWioqL44YcfKFOmDNOmTaNmzZpeh2aMSePSSy9l1KhRPPHEExmWe/LJJ3nsscfYsmULJUuWJCYmBoDZs2ezZcsWtmzZwvjx43nwwQcBSE5Opk+fPsyePZvY2FimTp1KbGxsRrvIMSxRBNhPP/1Eo0aNGDhwIH///Tf33XcfsbGx3HHHHV6HZkyGRALzOp/bb7+devXqERERwfjx41OXFy1aNHV62rRp3HfffYBz1WCHDh2IiooiKiqKpUuXXtTnvvzyy2nQoEGGl5eqKt9++y2dOnUCoGvXrkyfPh2AGTNm8J///AcRoXHjxhw+fJi9e/eycuVKqlatSuXKlcmfPz933XUXM2bMuKhYs0tQ9lEEU1/biBEj+PHHH7n66qsZP348LVu29DokY3K0iRMncumll3LixAkaNGjAHXfckeFNp3379qVZs2Z88cUXJCcnEx8ff1aZzp07s3nz5rOW9+vXj//85z8XHOPBgwe55JJLCAtzTqHlypVj9+7dAOzevZvy5U8Pc3dqXXrLV6xYccH79kJQJoqcLiEhgYIFCwIwfPhwrrjiCp555pkzvhEZk9N5NWjDqFGj+OKLLwDYtWsXW7ZsyTBRfPvtt0yePBmA0NBQSpQocVaZjz/+OEtjTG9Ei1NXG51rXUbvyeksUWSh+Ph4nnrqKRYuXMjq1avJnz8/pUqVCqpOK2O8tHDhQubPn8+yZcsoXLgwzZs3T70nwPekeqH3CWR1jeKyyy7j8OHDJCUlERYWRlxcXOpDw8qVK8euXacfxXNq3cmTJ9NdHgysjyKLzJ07l8jISEaPHk1sbCyLFy/2OiRjgs6RI0coWbIkhQsX5pdffmH58uWp68qUKcOmTZtISUlJrXEAtGjRgnfeeQdwOoyPHj161nY//vhjfvrpp7NemUkS4CStG2+8kWnTpgHw3nvv0b698wDPdu3aMXnyZFSV5cuXU6JECa688koaNGjAli1b2L59OydPnuSjjz6iXbt2mdp/tlPVoHpBPV2xQnOMgwcP6n333ac4D2XSOnXq6Nq1a70Oy5hMiY2N9XT/CQkJ2rp1a61Zs6Z26tRJmzVrpt99952qqn766adauXJlbdasmfbp00e7du2qqqq///67tmvXTiMjIzUqKkqXLl16UTHs3btXy5Ytq8WKFdMSJUpo2bJl9ciRI6qq2qZNG929e7eqqv7vf//TBg0aaJUqVbRTp06akJCgqqopKSnau3dvrVy5skZGRuqqVatSt/3VV19ptWrVtHLlyvriiy9eVJwZSe/3CKzWTJ53g3L02BUrVtOwodeRwKxZs3jggQfYt28fBQoUYMiQITz++OOpHVzGBJtNmzbZ8DG5QHq/Rxs91iMHDx5k3759XHfddUyYMIHq1at7HZIxxmQ5SxQXQFWJjY0lIiICcK6dLlGiBO3bt7fxmYwxuVZQnt28uKJsx44dtG7dmvr167NlyxY3DqFDhw6WJIwxuZqd4c4jJSWF0aNHExkZydy5cylcuDDbtm3zOixjjMk21vSUgU2bNhEdHZ06JMCdd97JqFGjKFOmjMeRGWNM9rFEcQ5Tpkzh/vvv5+TJk1xxxRW888473H777V6HZYwx2c6ans6hXr16hISE0L17d2JjYy1JGBOkKlasyIEDB3LdvrKTJQrXiRMniImJSR2PpXr16vz6669MmDCBkiVLehydMXmPqpKSkuJ1GAZLFAAsXryY2rVrEx0dzQcffJC63HekR2PyIhE558t3CPDx48dnWNZfO3bsoEaNGvTu3Zu6deuya9cuHnzwQerXr09ERASDBg1KLVuxYkUGDRpE3bp1qVmzJr/88gvg3N/UsmVL6tSpQ8+ePc8YjO+NN94gMjKSyMhIRowYkbrPa665hujoaCIjI7nnnnuYP38+TZs2pVq1aqxcufKsOJOTk3niiSeoWbMmtWrVYvTo0anrRo8efVZMK1eu5Nprr6VOnTpce+21qeNOTZo0iY4dO9K6dWuqVatG//79U7czZ84c6tatS1RUFC1atADg+PHjdOvWjQYNGlCnTp3sG6Y8s7d0e/WCeupzR/xFOXr0qPbu3Tt1+I0aNWrosmXLsmbjxgShtEM/nPrfSO81bty41HLjxo3LsKy/tm/friJyxv/hwYMHVVU1KSlJmzVrpuvWrVNV1auvvlpHjRqlqqpjxozR7t27q6rqww8/rEOGDFFV1S+//FIB3b9/v65evVojIyM1Pj5ejx07puHh4frjjz/q9u3bNTQ0VNevX6/Jyclat25dvf/++zUlJUWnT5+u7du3PyvOt99+Wzt27KiJiYlnxHiumI4cOZJadt68edqxY0dVVf3vf/+rlSpV0sOHD+uJEye0QoUKunPnTv3jjz+0XLlyum3btjO2P3DgQH3//fdVVfXQoUNarVo1jY+PPyu+rB7CI892Zs+ePZuePXuya9cuwsLCGDhwIE8//TQFChTwOjRjcgz1c4ifHj160KNHjyzZ59VXX03jxo1T5z/55BPGjx9PUlISe/fuJTY2llq1agHQsWNHwOlT/PzzzwFYtGhR6vStt96a2nT8ww8/0KFDB4oUKZL63sWLF9OuXTsqVaqU+sTJiIgIWrRogYhQs2ZNduzYcVaM8+fPp1evXqnD9Vx66aWp69KL6ciRI3Tt2pUtW7YgIiQmJqaWb9GiRerQ6OHh4fz2228cOnSIG264gUqVKp2x/blz5zJz5szUR7QmJCSwc+fOgA+7kicTxbRp0/jXv/4FOL/MiRMnpv7hGWO8depEDs5zqYcNG8aqVasoWbIk99133xlDjJ/6YhcaGkpSUlLq8vSauzJKer5fEENCQlLnQ0JCztiu77bO1aSWXkzPPvssN954I1988QU7duygefPm6e771HvOtX1V5bPPPsv24YLyZB/FbbfdRr169Rg6dCjLly+3JGFMDnX06FGKFClCiRIl2LdvH7Nnzz7ve2644QY+/PBDwGk5OHToUOry6dOn89dff3H8+HG++OILrr/++kzF1bJlS8aOHZuaCP78888Myx85coSyZcsCTr/E+TRp0oTvv/+e7du3n7H9Vq1aMXr06NSkt3bt2kzFf6HyRKLYs2cP3bp1Sz3YBQoUYMWKFTzxxBM20qsxOVhUVBR16tQhIiKCbt260bRp0/O+Z9CgQSxatIi6desyd+5cKlSoAEDdunW57777aNiwIY0aNSI6Opo6depkKq7o6GgqVKhArVq1iIqKYsqUKRmW79+/PwMHDqRp06YkJyefd/ulS5dm/PjxdOzYkaioKDp37gw4NZPExERq1apFZGQkzz77bKbiv1BBOcz4qlWrqe/HYLmqysSJE3n88cc5cuQIPXv2ZOzYsYEP0pggZcOM5w42zLiftm3bRo8ePViwYAHgdGo9/fTTHkdljDHBJ9c1PSUnJzNixAhq1qzJggULKFWqFB9++CGzZs2y+yKMMSYTgrJGkdH9O6tWreKxxx4DoEuXLowcOZLSpUtnU2TGBL+MrugxOV8guhOCMlGklZKSkvpMiMaNG/P000/TqFEjbrvtNo8jMya4FCxYkIMHD1KqVClLFkFIVTl48CAFCxbM0u0GZWf26tWrqVfPmV+9ejXR0dGMHj0605e6GWMciYmJxMXFnXGvggkuBQsWpFy5cuTLl++M5XmyM/vEiRMMGjSI4cOHk5KSwiuvvGKJwpiLlC9fvtS7gY05JaCd2SLSWkQ2i8hWERmQznoRkVHu+vUiUtef7a5Z8z21atVi6NChADz++ONMmzYti6M3xhgDAWx6EpFQ4Ffgn0AcsArooqqxPmVuAR4GbgEaASNVtVHG271cYT8AkZGRxMTE0LBhw4B8BmOMyS0upukpkDWKhsBWVd2mqieBj4D2acq0Bya7gxsuBy4RkSsz3uwhwsLyMXjwYNasWWNJwhhjAiyQfRRlgV0+83E4tYbzlSkL7PUtJCI9gFNDU/6dlMSGwYMHM3jw4CwNOAhdBuS+x2lljh2L0+xYnGbH4rRMjyQYyESR3rV1adu5/CmDqo4HxgOIyOrMVp9yGzsWp9mxOM2OxWl2LE4TkdWZfW8gm57iAN9bocsBezJRxhhjjIcCmShWAdVEpJKI5AfuAmamKTMT+I979VNj4Iiq7k27IWOMMd4JWNOTqiaJyEPAN0AoMFFVN4pIL3f9WOBrnCuetgJ/Aff7senx5y+SZ9ixOM2OxWl2LE6zY3Fapo9F0N2ZbYwxJnvlutFjjTHGZC1LFMYYYzKUYxNFoIb/CEZ+HIt73GOwXkSWikiUF3Fmh/MdC59yDUQkWUQ6ZWd82cmfYyEizUXkJxHZKCLfZ3eM2cWP/5ESIjJLRNa5x8Kf/tCgIyITReQPEdlwjvWZO2+qao574XR+/w+oDOQH1gHhacrcAszGuRejMbDC67g9PBbXAiXd6TZ5+Vj4lPsW52KJTl7H7eHfxSVALFDBnb/c67g9PBZPAa+506WBP4H8XscegGNxA1AX2HCO9Zk6b+bUGkWAhv8ISuc9Fqq6VFUPubPLce5HyY38+bsAZ/ywz4A/sjO4bObPsbgb+FxVdwKoam49Hv4cCwWKifOQjaI4iSIpe8MMPFVdhPPZziVT582cmijONbTHhZbJDS70c3bH+caQG533WIhIWaADMDYb4/KCP38X/wBKishCEVkjIv/Jtuiylz/H4i2gBs4NvT8Dj6hqSvaEl6Nk6ryZU59HkWXDf+QCfn9OEbkRJ1FcF9CIvOPPsRgBPKmqybn8CW3+HIswoB7QAigELBOR5ar6a6CDy2b+HItWwE/ATUAVYJ6ILFbVo4EOLofJ1HkzpyYKG/7jNL8+p4jUAiYAbVT1YDbFlt38ORb1gY/cJHEZcIuIJKnq9OwJMdv4+z9yQFWPA8dFZBEQhTP8f27iz7G4H3hVnYb6rSKyHbgGWJk9IeYYmTpv5tSmJxv+47TzHgsRqQB8DtybC78t+jrvsVDVSqpaUVUrAtOA3rkwSYB//yMzgOtFJExECuOM3rwpm+PMDv4ci504NStEpAzOSKrbsjXKnCFT580cWaPQwA3/EXT8PBbPAaWAt91v0kmaC0fM9PNY5An+HAtV3SQic4D1QAowQVXTvWwymPn5d/ECMElEfsZpfnlSVXPd8OMiMhVoDlwmInHAICAfXNx504bwMMYYk6Gc2vRkjDEmh7BEYYwxJkOWKIwxxmTIEoUxxpgMWaIwxhiTIUsUJkcTkb4isklEPsygTHMR+TI74zoXEWl3avRSEbldRMJ91j0vIjdnYyzNReTa7Nqfyb1y5H0UxvjojXO3+XavA/GHqs7k9M1etwNf4ozgiqo+l9X7E5EwVT3X4HbNgXhgaVbv1+QtVqMwOZaIjMUZOnqmiDwmIg3d522sdX9WT+c9zdznL/zklivmLv8/EVnljsE/5Bz7ixeR4SLyo4gsEJHS7vLaIrLcfe8XIlLSXd5XRGLd5R+5y+4Tkbfcb/LtgKFuLFVEZJKIdBKRNiLyic9+m4vILHe6pYgsc2P4VESKphPnQhF5WZznSzwiIreJyAr3884XkTIiUhHoBTzm7v96ESktIp+5x2GViDS9iF+PyUu8Hj/dXvbK6AXsAC5zp4sDYe70zcBn7nRz4Et3ehbQ1J0uilNrbonzYHnB+XL0JXBDOvtS4B53+jngLXd6PdDMnX4eGOFO7wEKuNOXuD/v83nfJHyeh3Fq3o1pJ1DEXf4O8G+csakW+Sx/EngunTgXAm/7zJfk9M2z0cBwd3ow8IRPuSnAde50BWCT179fewXHy5qeTDApAbwnItVwTur50imzBHjD7dP4XFXjRKQlTrJY65YpClTDOSn7SgE+dqc/AD4XkRI4SeDU0+HeAz51p9cDH4rIdMDv8aTUGXJiDnCbiEwDbgX6A82AcGCJOxRLfmDZOTbzsc90OeBjcZ4rkB84VzPdzUC4nB5Vt7iIFFPVY/7GbvImSxQmmLwAfKeqHdymlYVpC6jqqyLyFc54NsvdzmMBXlHVcRe4v/ONb3MrzhPF2gHPikjEBWz7Y6APzkNmVqnqMXHO4PNUtYsf7z/uMz0aeENVZ4pIc5yaRHpCgCaqeuIC4jTG+ihMUCkB7Han70uvgIhUUdWfVfU1YDXOUNLfAN1OtfeLSFkRuTydt4fgNA2B83S4H1T1CHBIRK53l98LfC8iIUB5Vf0OpzZwCU5NxdcxoNg5PstCnEdWPsDp2sFyoKmIVHXjLCwi/zjH+335HpeuGex/LvDQqRkRqe3Hto2xRGGCyuvAKyKyBGeU0PQ8KiIbRGQdcAKYrapzcdrnl7mjh04j/RP4cSBCRNbgPODmeXd5V5xO6fVAbXd5KPCBu721wJuqejjN9j4C/s/tZK7iu0JVk3H6Stq4P1HV/TgJcKq7r+U4ie58BgOfishiwHdE1FlAh1Od2UBfoL7b+R6L09ltzHnZ6LHGuEQkXlXPusrImLzOahTGGGMyZDUKY4wxGbIahTHGmAxZojDGGJMhSxTGGGMyZInCGGNMhixRGGOMydD/AyD9TwdGZbS8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot ROC\n",
    "predict = best_model.predict(testimages)\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "fpr, tpr, thresholds = roc_curve(testlabels, predict)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "plt.plot(fpr, tpr, lw=2, color='b', label='auc = %.3f' % (roc_auc))\n",
    "plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='k', label='random chance')\n",
    "plt.xlim([0, 1.0])\n",
    "plt.ylim([0, 1.0])\n",
    "plt.xlabel('false positive rate')\n",
    "plt.ylabel('true positive rate')\n",
    "plt.title('receiver operating curve')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAWVklEQVR4nO3dfXBV9Z3H8feXEMwqoBbQAQJJLKBkCLjsFRh0FdCxUOWpSy0Ia9laMqC0+48dlNlZ23G21Y5/7DjrQ/FhqI6DpVhA8LEtIDLQyg0gD6a0rCUYwRrAyRYVaeC7fySkMQZybu7Dyf3l85q5M9xz7jnn+7vJ/eTwO797fubuiIhIWLrFXYCIiGSewl1EJEAKdxGRACncRUQCpHAXEQlQ97gLAOjbt6+XlpbGXYaISF6pqqo66u792lrXKcK9tLSUZDIZdxkiInnFzGrOtU7dMiIiAVK4i4gESOEuIhKgTtHnLiL54W9/+xu1tbWcPHky7lK6lKKiIoqLiyksLIy8jcJdRCKrra2lV69elJaWYmZxl9MluDvHjh2jtraWsrKyyNtlpVvGzC4ysyozuzUb+xeReJw8eZI+ffoo2HPIzOjTp0/K/1uKFO5m9oyZfWRme1stn2xm+83sgJnd22LVEmBlSpWISF5QsOdeR97zqGfuy4HJrQ5WADwKTAHKgTlmVm5mNwHvAn9JuRoREcmISOHu7puB460WjwEOuPt77n4KeAGYDkwExgG3AwvMrM1jmFmlmSXNLFlXV9fhBlBaCmbxPPStWunqMv35a+czdfDgQUaMGJFWyZs2beLWWztnj/H8+fNZtWpVRvaVzgXVgcD7LZ7XAmPdfTGAmc0Hjrr7mbY2dvdlwDKARCLR8RlDamogrglH9N9T6eoy/fmL6TO1ezecOtX+69wdd6dbt8xdruzRA0aOzNjumqUT7m39FJp/yu6+vN0dmE0Fpg4ZMiSNMkSkK2loaODb3/42O3fuZNiwYTz77LM8/PDDrFu3js8++4zx48fzs5/9DDPjwIEDLFy4kLq6OgoKCvjlL3/5hX1t376dyspK7r//Ra69the33347x44d45prruG1116jqqqKEydOMGXKFCZOnMi2bdtYs2YNW7du5cc//jHuzi233MJDDz0EQM+ePTlx4gQAq1atYv369Sxfvpz58+fTu3dvkskkH374IT/96U+ZNWsW7s63vvU99u7dQFlZGZmcGS+dPz+1wKAWz4uBw6nswN3XuXvlxRdfnEYZItKV7N+/n8rKSnbv3k3v3r157LHHWLx4Mdu3b2fv3r189tlnrF+/HoC5c+dy9913884777B161b69+/fvJ+tW7eycOFC1q5dS3HxFfzoRz9i0qRJ7Nixg5kzZ3Lo0KEvHPOOO+5g586dFBYWsmTJEjZs2MCuXbvYvn07a9asabfuI0eOsGXLFtavX8+99zaOP1m9ejU1NfvZs2cPTz75JFu3bs3Y+5ROuG8HhppZmZn1AGYDL2WmLBGRtg0aNIhrr70WgHnz5rFlyxY2btzI2LFjqaioYMOGDezbt4+//vWvfPDBB8ycORNo/CLQhRdeCEB1dTWVlZWsW7eOwYMHA7BlyxZmz54NwOTJk7n00kubj1lSUsK4ceOAxrP9CRMm0K9fP7p3787cuXPZvHlzu3XPmDGDbt26UV5ezl/+0jjeZPPmzXzta3MoKChgwIABTJo0KUPvUvShkCuAbcCVZlZrZne6ewOwGHgdqAZWuvu+VA5uZlPNbFl9fX2qdYtIF9V6WKCZcdddd7Fq1Sr27NnDggULOHny5Hm7OPr3709RURE7d+5sXna+11900UWRXteyttbj0i+44IJz7CM71xmijpaZ4+793b3Q3Yvd/emm5a+4+zB3/6q7/1eqB1e3jIik6tChQ2zbtg2AFStWcN111wHQt29fTpw40TzapHfv3hQXFzd3mXz++ed8+umnAFxyySW8/PLLLF26lE2bNgFw3XXXsXJl49dz3njjDT7++OM2jz927FjefPNNjh49yunTp1mxYgU33HADAJdffjnV1dWcOXOG1atXt9uW66+/nl//+gVOnz7NkSNH2LhxYwfflS/TjcNEpONKSjI7FLKkpN1DDh8+nJ///OeMHDmS48ePs2jRIhYsWEBFRQUzZszgmmuuaX7tc889xyOPPMLIkSMZP348H374YfO6yy+/nHXr1nH33Xezd+/vuf/++3njjTcYPXo0r776Kv3796dXr15fOn7//v35yU9+wsSJExk1ahSjR49m+vTpADz44IPceuutTJo06Qv9++cyc+ZMBg0aSkVFBYsWLWr+I5EJlsmrsykf/O+jZRb86U9/6uhO4h0KGeP7J5Jr1dXVDB8+PO4yMi6ZhIqKzykoKKB79+5s27aNRYsWsWvXrpwcO5Fo/3VtvfdmVuXubW4d643D3H0dsC6RSCyIsw4RkUOHDnHbbbdx5swZevTowZNPPhl3SWnRXSFFRIChQ4d+4QJrvou1z12jZUREsiPWcNdoGRGR7NBoGRGRACncRUQCpD53EemwHN/xt03f/e53effddzPdNHr27JnxfeaShkKKSId1hjv+PvXUU5krICDqlhGRvPHJJ59wyy23MGrUKEaMGMEvfvELJkyYQDKZBODpp59m2LBhTJgwgQULFrB48WKgcRKM73//+4wfP54rrrii+RYFJ06c4MYbb2TevNFUVFSwdu3a2NqWaRrnLiJ547XXXmPAgAG8/PLLANTX1/P4448DcPjwYR544AF27NhBr169mDRpEqNGjWre9uwtd//whz8wbdo0Zs2aRVFREatXr+aPf+xNaelRxo0bx7Rp04KYJ1Zn7iKSNyoqKvjNb37DkiVLeOutt2g5jPrtt9/mhhtu4Ctf+QqFhYV885vf/MK2bd1y191ZunQpc+aM5KabbuKDDz5oXpfvYj1z10xMIpKKYcOGUVVVxSuvvMJ9993HzTff3LyuvftktXXL3eeff566ujqee66KceMKKS0t/dKtevOVvsQkInnj8OHDXHjhhcybN4977rmHHTt2NK8bM2YMb775Jh9//DENDQ28+OKL7e6vvr6eyy67jO7dC9m4cSM1NTXZLD+n1OcuIh129o6/mdzf+ezZs4cf/OAHdOvWjcLCQh5//HHuueceAAYOHMjSpUsZO3YsAwYMoLy8nPZOHOfOncvUqVP57W8TjB9/NVdddVWmmhK7WG/5e1YikfCzV7tTplv+iuRMZ7/l74kTJ+jZsycNDQ3MnDmT73znO83T7J1P1NvuZkO2bvmrC6oiEowf/vCHXH311YwYMYKysjJmzJgRd0mxUbeMiATj4YcfjruETkO3HxCRlHSGrtyupiPvuUbLiEhkRUVFHDt2TAGfQ+7OsWPHKCoqSmk7dcuISGTFxcXU1tZSV1cXdykZdfQoVFd33mMXFRVRXFyc0n4V7iISWWFhIWVlZXGXkXHl5fENfMvWsTVaRkQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQPoSk4hIgPQlJhGRAKlbRkQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQBkPdzMbbmZPmNkqM1uU6f2LiEj7IoW7mT1jZh+Z2d5Wyyeb2X4zO2Bm9wK4e7W7LwRuAxKZL1lERNoT9cx9OTC55QIzKwAeBaYA5cAcMytvWjcN2AL8NmOViohIZJHC3d03A8dbLR4DHHD399z9FPACML3p9S+5+3hgbiaLFRGRaNKZIHsg8H6L57XAWDObAHwDuAB45Vwbm1klUAkwePDgNMoQEZHW0gl3a2OZu/smYFN7G7v7MmAZQCKRiGnecRGRMKUzWqYWGNTieTFwOJUdaLIOEZHsSCfctwNDzazMzHoAs4GXUtmBJusQEcmOqEMhVwDbgCvNrNbM7nT3BmAx8DpQDax0932pHFxn7iIi2WHu8Xd3JxIJTyaTHdvYDOJqQ5zHFpGMydcYMbMqd2/z+0S6/YCISIBiDXd1y4iIZEes4a4LqiIi2aFuGRGRACncRUQCpD53EZEAqc9dRCRA6pYREQmQumVERAKkbhkRkQCpW0ZEJEAKdxGRACncRUQCpAuqIiIB0gVVEZEAqVtGRCRACncRkQAp3EVEAqRwFxEJkEbLiIgESKNlREQCpG4ZEZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZC+xCQiEiB9iUlEJEDqlhERCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQFkJdzObYWZPmtlaM7s5G8cQEZFzixzuZvaMmX1kZntbLZ9sZvvN7ICZ3Qvg7mvcfQEwH/hWRisWEZF2pXLmvhyY3HKBmRUAjwJTgHJgjpmVt3jJfzStFxGRHIoc7u6+GTjeavEY4IC7v+fup4AXgOnW6CHgVXff0db+zKzSzJJmlqyrq+to/SIi0oZ0+9wHAu+3eF7btOx7wE3ALDNb2NaG7r7M3RPunujXr1+aZYiISEvd09ze2ljm7v4I8Ei7G5tNBaYOGTKkwwWU8mdq2qoiB0r4MwfjObSIyHmle+ZeCwxq8bwYOBx140xM1lFDKe7E8qihtMN1i4hkU7rhvh0YamZlZtYDmA28lH5ZIiKSjlSGQq4AtgFXmlmtmd3p7g3AYuB1oBpY6e77Utin5lAVEckCc/e4ayCRSHgymezQtmaNXSRxiPPYIpI5+ZojZlbl7om21un2AyIiAYo13NUtIyKSHbGGeyZGy4iIyJepW0ZEJEDqlhERCZC6ZUSk0ygtbRw9kutHSUncLc+8dG8/ICKSMTU1Gl6cKepzFxEJkPrcRUQCpD53EZEAqVtGRCRACncRkQAp3EVEAqQLqiIiAdIFVRGRAKlbRkQkQAp3EZEAKdxFRAKkcBcRCZBGy4iIBEijZUREAqRuGRGRACncRc4jrskjzBqP3dXaHOKkGXHRZB0i5xHn5BFnQzbXSko0YUYIFO55qLS0MXTiUFICBw/Gc+yuRu+zpEPhnofiPJuM40xSRFKnPncRkQAp3CUvxHWRTxf4JF/F2i1jZlOBqUOGDImzDMkDcXZFSRcQ54UssvOLrS8xiYicPXuI45EluqAqKSkpiW94nohEp3CXlGh4nkh+0AVVEZEAKdxFRAKkbpl0xfKtHg0bEZHz05l7GkpKwPCcP0o4GHfTRaST05l7GmK7uGhl6OxdRM5HZ+4iIgFSuIuIBCjj4W5mV5jZ02a2KtP7FhGRaCKFu5k9Y2YfmdneVssnm9l+MztgZvcCuPt77n5nNooVEZFoop65Lwcmt1xgZgXAo8AUoByYY2blGa1OREQ6JFK4u/tm4HirxWOAA01n6qeAF4DpUQ9sZpVmljSzZF1dXeSCRUSkfen0uQ8E3m/xvBYYaGZ9zOwJ4B/N7L5zbezuy9w94e6Jfv36pVGGiIi0ls4497a+munufgxYmMZ+RUQkTemcudcCg1o8LwYOp7IDM5tqZsvq6+vTKENERFpLJ9y3A0PNrMzMegCzgZdS2YEm6xARyY6oQyFXANuAK82s1szudPcGYDHwOlANrHT3fakcXGfuIiLZYd4JJqZMJBKeTCY7tK1ZF5xbs0s2WiSLYvxMpXNoM6ty90Rb63T7ARGRAMUa7uqW6aCzE5nG8Sgtjbv1IhJBrOGuC6oddPBgfDO119TE3XoRiUDdMiIiAVK3jIhIgNQtIyISIHXLiIgESOEuIhIg9blLauIahqkhmCIpUZ+7pCauYZgagimSEnXLiIgESOEuIhIghbuISIB0QVVEOo/S0ngu2JeUxN3yjNMFVRHpPGpq4rlgf/Bg3C3POHXLiIgESOEuIhIghbuISIAU7iIiAdJoGZHziWv0hm65IGnSaBmR84lr9IZuuSBpUreMiEiAFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgFSuIuIBKh7nAc3s6nA1CFDhsRZhuSDs3O3xnHcrqa0NL4x9l3x/c4Sc/e4ayCRSHgymezQtmaN3/cQCU5cv9z6UOVUOm+3mVW5e6KtdeqWEREJkMJdRCRACncRkQAp3EVEAqRwFxEJkMJdRCRACncRkQAp3EVEAqRwFxEJkMJdRCRAGb+3jJldBDwGnAI2ufvzmT6GiIicX6QzdzN7xsw+MrO9rZZPNrP9ZnbAzO5tWvwNYJW7LwCmZbheERGJIGq3zHJgcssFZlYAPApMAcqBOWZWDhQD7ze97HRmyhQRkVRECnd33wwcb7V4DHDA3d9z91PAC8B0oJbGgD/v/s2s0sySZpasq6tLvXKR0J29zXGuH7rtbhDSuaA6kL+foUNjqA8EfgX8i5k9Dqw718buvszdE+6e6NevXxpliATq4MHGe8Hm+nHwYNwtlwxI54JqWzMnuLt/AvxbpB1osg4RkaxI58y9FhjU4nkxcDiVHbj7OnevvPjii9MoQ0REWksn3LcDQ82szMx6ALOBlzJTloiIpCPqUMgVwDbgSjOrNbM73b0BWAy8DlQDK919XyoHN7OpZrasvr4+1bpFROQ8NIeqiEiMNIeqiIhEFmu4q1tGRCQ7Yg13jZYREcmOTtHnbmZ1QE0HN+8LHM1gOflAbe4a1OauIZ02l7h7m98C7RThng4zS57rgkKo1OauQW3uGrLVZl1QFREJkMJdRCRAIYT7srgLiIHa3DWozV1DVtqc933uIiLyZSGcuYuISCsKdxGRAOVNuJ9jvtaW683MHmlav9vMRsdRZyZFaPPcprbuNrOtZjYqjjozqb02t3jdNWZ22sxm5bK+bIjSZjObYGa7zGyfmb2Z6xozKcLv9cVmts7M3mlqb6T5ITqzc81D3WJ95vPL3Tv9AygA/he4AugBvAOUt3rN14FXaZxEZBzw+7jrzkGbxwOXNv17Sldoc4vXbQBeAWbFXXcOfs6XAO8Cg5ueXxZ33Vlu71LgoaZ/96Nxis8ecdeeZruvB0YDe8+xPuP5lS9n7uear7Wl6cCz3uh3wCVm1j/XhWZQu212963u/nHT09/x97lr81WUnzPA94AXgY9yWVyWRGnz7cCv3P0QgLvnc7ujtNeBXmZmQE8aw70ht2Vmlrc9D3VLGc+vfAn3c83Xmupr8kmq7bmTxr/8+azdNpvZQGAm8EQO68qmKD/nYcClZrbJzKrM7I6cVZd5Udr7P8BwGmd22wP8u7ufyU15scl4fqUzh2outTlfawdek08it8fMJtIY7tdltaLsi9Lm/waWuPvpxhO7vBelzd2BfwJuBP4B2GZmv3P3P2a7uCyI0t6vAbuAScBXgV+b2Vvu/n/ZLi5GGc+vfAn3KPO1pj2naycTqT1mNhJ4Cpji7sdyVFu2RGlzAnihKdj7Al83swZ3X5ObEjMu6u/2UW+cfP4TM9sMjALyMdyjtPffgAe9sTP6gJn9GbgKeDs3JcYi4/mVL90yUeZrfQm4o+mq8zig3t2P5LrQDGq3zWY2GPgV8K95ehbXWrttdvcydy9191JgFXBXHgc7RPvdXgv8s5l1N7MLgbE0Tm2Zj6K09xCN/0vBzC4HrgTey2mVuZfx/MqLM3d3bzCzs/O1FgDPuPs+M1vYtP4JGkdOfB04AHxK41//vBWxzf8J9AEeazqTbfA8vqNexDYHJUqb3b3azF4DdgNngKfcvc0hdZ1dxJ/xA8ByM9tDY3fFEnfP69sAW+M81BOAvmZWC9wPFEL28ku3HxARCVC+dMuIiEgKFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgFSuIuIBOj/AX25UYtvONnqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist, bin_edges = np.histogram(predict[:int(len(predict)/2)])\n",
    "hist_, bin_edges_ = np.histogram(predict[:-int(len((predict)-1)/2)])\n",
    "plt.hist([predict[:int(len(predict)/2)].flatten(), predict[int(len(predict)/2)-1:-1].flatten()] ,\n",
    "         color = ['blue', 'red'], histtype = 'step', label = ['signal', 'background'])\n",
    "plt.legend()\n",
    "plt.yscale('log')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}