2022_10_07_Zahmeeth_Sakkaff_Session3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMprYCP9Sn8tjp0iYsXCD/G",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zahmeeth/ai-science-training-series/blob/main/2022_10_07_Zahmeeth_Sakkaff_Session3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Home Work Session 3: Neural Networks in Tensorflow (10/07/22) by Zahmeeth Sakkaff\n",
        "\n",
        "CIFAR-10 dataset classification with CNNs"
      ],
      "metadata": {
        "id": "fjo9pccSYUjl"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Homework: improve the accuracy of this model\n",
        "\n",
        "I have changed the optimizer from sgd to adamax, lowering the batch_size 64, increasing the epochs 6 and correct lr would improve accuracy of the model where it increased from 58.69% to 66.25% with trial an error or all combination of these parameters can improve the accuracy"
      ],
      "metadata": {
        "id": "Q4sy3js9OWyT"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Ynl_PRykXH4W"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy\n",
        "import matplotlib.pyplot as plt\n",
        "import time"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "CIFAR-10 data set"
      ],
      "metadata": {
        "id": "IcODTxmQYwdz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
        "x_train = x_train.astype(numpy.float32)\n",
        "x_test  = x_test.astype(numpy.float32)\n",
        "\n",
        "x_train /= 255.\n",
        "x_test  /= 255.\n",
        "\n",
        "y_train = y_train.astype(numpy.int32)\n",
        "y_test  = y_test.astype(numpy.int32)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0z64gzTfY0LK",
        "outputId": "245257de-a4ba-4cd5-d24d-be9ff820a5f4"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 2s 0us/step\n",
            "170508288/170498071 [==============================] - 2s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convolutional neural network (CNN)"
      ],
      "metadata": {
        "id": "x6NqWcvnatCj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CIFAR10Classifier(tf.keras.models.Model):\n",
        "\n",
        "    def __init__(self, activation=tf.nn.tanh):\n",
        "        tf.keras.models.Model.__init__(self)\n",
        "\n",
        "        self.conv_1 = tf.keras.layers.Conv2D(32, [3, 3], activation='relu')\n",
        "        self.conv_2 = tf.keras.layers.Conv2D(64, [3, 3], activation='relu')\n",
        "        self.pool_3 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))\n",
        "        self.drop_4 = tf.keras.layers.Dropout(0.25)\n",
        "        self.dense_5 = tf.keras.layers.Dense(128, activation='relu')\n",
        "        self.drop_6 = tf.keras.layers.Dropout(0.5)\n",
        "        self.dense_7 = tf.keras.layers.Dense(10, activation='softmax')\n",
        "\n",
        "    def call(self, inputs):\n",
        "\n",
        "        x = self.conv_1(inputs)\n",
        "        x = self.conv_2(x)\n",
        "        x = self.pool_3(x)\n",
        "        x = self.drop_4(x)\n",
        "        x = tf.keras.layers.Flatten()(x)\n",
        "        x = self.dense_5(x)\n",
        "        x = self.drop_6(x)\n",
        "        x = self.dense_7(x)\n",
        "\n",
        "        return x"
      ],
      "metadata": {
        "id": "cMOxizt5aseR"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Simple training"
      ],
      "metadata": {
        "id": "14zFDCWRa4Ab"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train_network_concise(_batch_size, _n_training_epochs, _lr):\n",
        "\n",
        "    cnn_model = CIFAR10Classifier()\n",
        "\n",
        "    cnn_model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adamax\", metrics=['accuracy'])\n",
        "    \n",
        "    history = cnn_model.fit(x_train, y_train, batch_size=_batch_size, epochs=_n_training_epochs)\n",
        "    return history, cnn_model"
      ],
      "metadata": {
        "id": "eg6NGJTma4yJ"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This took 43 seconds per epoch on my laptop\n",
        "batch_size = 64\n",
        "epochs = 6\n",
        "lr = .05\n",
        "history, cnn_model = train_network_concise(batch_size, epochs, lr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n0ScfertbIjc",
        "outputId": "1f23680d-7a79-4d0a-d999-19da98b509c7"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/6\n",
            "782/782 [==============================] - 169s 215ms/step - loss: 1.6876 - accuracy: 0.3914\n",
            "Epoch 2/6\n",
            "782/782 [==============================] - 169s 216ms/step - loss: 1.3833 - accuracy: 0.5045\n",
            "Epoch 3/6\n",
            "782/782 [==============================] - 168s 215ms/step - loss: 1.2612 - accuracy: 0.5558\n",
            "Epoch 4/6\n",
            "782/782 [==============================] - 168s 215ms/step - loss: 1.1715 - accuracy: 0.5869\n",
            "Epoch 5/6\n",
            "782/782 [==============================] - 170s 217ms/step - loss: 1.0979 - accuracy: 0.6127\n",
            "Epoch 6/6\n",
            "782/782 [==============================] - 168s 215ms/step - loss: 1.0433 - accuracy: 0.6338\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Accuracy for test data. The model should be better than the non-convolutional model even if you're only patient enough for three epochs."
      ],
      "metadata": {
        "id": "Pi4hm0b-MR7E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(5,3))\n",
        "plt.plot(history.epoch,history.history['loss'])\n",
        "plt.title('loss')\n",
        "\n",
        "plt.figure(figsize=(5,3))\n",
        "plt.plot(history.epoch,history.history['accuracy'])\n",
        "plt.title('accuracy');"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "PvONw0gNMVqQ",
        "outputId": "2e439421-5695-46e7-de42-1c5afd3620f3"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 360x216 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAADSCAYAAADXPHxAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaGElEQVR4nO3deXhV9Z3H8fc3KwQSIISEsCQhsgsBJBIUiii1BRV3W5BaZ7R1dFS0WquO1ra2Mx3FWsQZbR1rrY8I7joF9xUQCWsIyCYEAgmBQMISCCHbb/7I1UELJOTe5Nyb+3k9Tx6T3MM5n/P48OFsv/Mz5xwiIuEgwusAIiKtRYUnImFDhSciYUOFJyJhQ4UnImFDhSciYUOFJ54xs21m9l2vc0j4UOGJSNhQ4YlI2FDhiefMLNbMZprZTt/XTDOL9X2WZGbzzGy/mZWb2UIzi/B9dreZFZtZhZltNLMJ3u6JBLsorwOIAPcBo4HhgAPeBO4HfgncCRQB3XzLjgacmQ0AbgHOdM7tNLMMILJ1Y0uo0RGeBINpwIPOuVLn3B7gN8A1vs9qgFQg3TlX45xb6BoGgNcBscBgM4t2zm1zzm3xJL2EDBWeBIMeQOExPxf6fgcwA9gMvGdmBWZ2D4BzbjNwO/BroNTM5ppZD0ROQoUnwWAnkH7Mz2m+3+Gcq3DO3emcywQuBu746lqdc+4F59xY3591wEOtG1tCjQpPgsEc4H4z62ZmScADwPMAZnaRmfU1MwMO0HAqW29mA8zsPN/NjSrgCFDvUX4JESo8CQa/A5YD+cAaYKXvdwD9gA+AQ8DnwBPOuY9puH73n8BeYBeQDNzburEl1JheACoi4UJHeCISNlR4IhI2VHgiEjZUeCISNlR4IhI2PBtLm5SU5DIyMrzavIi0UStWrNjrnOt2vM8aLTwzewa4CCh1zg05zud30TAW8qv1DQK6OefKT7bejIwMli9f3tjmRUROiZkVnuizppzSPgtMPNGHzrkZzrnhzrnhNDz4+WljZSci4oVGC885twBoaoFNpWGYkIhI0AnYTQszi6PhSPDVkyxzg5ktN7Ple/bsCdSmRUSaJJB3aScDn53sdNY595RzLts5l92t23GvKYqItJhAFt4UWuh0traunhnvbuDNvOKWWL2IhImAFJ6ZdQLOoeHV3C0it6Cc+15fy/ayypbahIi0cY0WnpnNoeG1PAPMrMjMrjezG83sxmMWuwx4zzl3uCVCRkVGMHPKcMxg+txV1NTptWcicuo8ez1Udna2O9Xn8Obnl3DzCyv51/Gn8YuJA1somYiEMjNb4ZzLPt5nITW07MKsVKac2ZsnP93C4s17vY4jIiEmpAoP4IHJg8lM6sDtL+ZRfrja6zgiEkJCrvDiYqKYNXUE+ytr+MUrq9Ebm0WkqUKu8ABO79GJeyYN5IP1pTz3+QmHzYmIfENIFh7AP4/J4LyByfz7W+tZX3LQ6zgiEgJCtvDMjBlXZtGpfTS3zlnFkeo6ryOJSJAL2cID6Noxlj/+YDhb9hzit/PXeR1HRIJcSBcewNh+SdwwLpMXcrfzztoSr+OISBAL+cIDuPP8AWT16sTdr65h5/4jXscRkSDVJgovJiqCWVNGUFtXz+1z86ir16MqIvKP2kThAWQkdeC3lw5h6bZy/uujzV7HEZEg1GYKD+DyM3px2YiePPbhJpZv01vmReSb2lThATx4yen06hLHbXPzOFBZ43UcEQkiba7w4ttFM2vqCHYfrOLfXl+joWci8rWmvA/vGTMrNbO1J1lmvJnlmdkXZvZpYCOeuuG9O3Pn9wYwf00JLy7b4XUcEQkSfk/TaGadgSeAi51zpwNXBSaaf/5lXCZj+ybxm7+vY3NphddxRCQIBGKaxquB15xz233LlwYom18iIoxHfzCM9jGR3Donj6oaDT0TCXeBuIbXH+hiZp+Y2Qoz+/GJFmztaRqTE9ox48os1pcc5KF3NrT49kQkuAWi8KKAkcCFwPeBX5pZ/+Mt6MU0jRMGpfBPZ2fw18+28dGG3a2yTREJToEovCLgXefcYefcXmABMCwA6w2YeyYNZFBqAj9/OZ/Sg1VexxERjwSi8N4ExppZlJnFATnA+gCsN2DaRUfy+NThVFbXcsdLq6nX0DORsOT3NI3OufXAO0A+sBR42jl3wkdYvNI3OZ5fTT6dRZv38tTCAq/jiIgHohpbwDk3tQnLzABmBCRRC5pyZm8WfrmHR97dyFmZXRnWu7PXkUSkFbW5kRYnY2b8/rIsUhLaMX3uKg4drfU6koi0orAqPIBOcdHMnDKcHeWVPPBG0J15i0gLCrvCAzgzI5HpE/rx2qpiXl9V5HUcEWklYVl4ALec25czM7pw/+trKSw77HUcEWkFYVt4UZERzJwygsgIY/qcVVTX1nsdSURaWNgWHkDPzu156IosVhcd4NH3N3kdR0RaWFgXHsCkoalMHZXGnz7dwqIv93odR0RaUNgXHsADFw2mb3JHfvZSHmWHjnodR0RaiAoPaB8TyawpIzhwpIa7XsnXW5JF2igVns/gHgn826SBfLShlGcXb/M6joi0ABXeMa49O4MJA5P5/VsbWLfzoNdxRCTAVHjHMDNmXDWMznHR3DpnJZXVGnom0pao8L4lsUMMf/zhcAr2Hua389Z5HUdEAkiFdxxj+iZx4zmnMWfpDt5aU+J1HBEJEL+nafRN0XjAN01jnpk9EPiYre+O8/szrHdn7nk1n6J9lV7HEZEA8HuaRp+Fzrnhvq8H/Y/lvejICGZNGU69g9vn5lFbp6FnIqEuENM0tlnpXTvwu0uHsLxwH49/tNnrOCLip0BdwzvLzFab2dtmdvqJFmrtaRoD4dIRPbn8jJ48/tGXLN0alr0v0mYEovBWAunOuWHA48AbJ1rQi2kaA+HBS4aQlhjH7XNXcaCyxus4ItJMfheec+6gc+6Q7/u3gGgzS/I7WRDpGBvFrKkjKK04yj2vaeiZSKjyu/DMrLuZme/7Ub51lvm73mCT1aszd31/AG+v3cWcpTu8jiMizdDorGW+aRrHA0lmVgT8CogGcM79CbgSuMnMaoEjwBTXRg+BfvqdTBZt3suD877gzIwu9EuJ9zqSiJwC86qbsrOz3fLlyz3Ztj9KK6qYNHMh3eJjeePmMbSLjvQ6kogcw8xWOOeyj/eZRlqcouT4djxy1TA27KrgP9/e4HUcETkFKrxmOHdgMteN6cOzi7fxwbrdXscRkSZS4TXT3ZMGMDg1gbteWc3ug1VexxGRJlDhNVNsVCSzpo6gqqaen72YR119m7xPI9KmqPD80De5I7++eDCLt5Tx5wVbvI4jIo1Q4fnpB9m9uTArlUff28Sq7fu8jiMiJ6HC85OZ8R+XDSUloR3T566iokpDz0SClQovADq1j2bW1OHs3F/F/W+s1dAzkSClwguQkemJ3DahH2/m7eS1lcVexxGR41DhBdDN5/ZlVJ9EHnhzLVv3HvY6joh8iwovgCIjjJk/HE5UZATT56yiulZvSRYJJiq8AOvRuT0PXZHFmuID/OG9jV7HEZFjqPBawMQh3ZmWk8afFxSwYFNovNlZJByo8FrI/RcOpl9yR+54aTV7Dx31Oo6IEIBpGo9Z7kwzqzWzKwMXL3S1j4nk8atHcLCqhp+/vJp6DT0T8VxApmk0s0jgIeC9AGRqMwZ2T+D+CwfxycY9/HXxNq/jiIS9QE3TeCvwKlAaiFBtyTWj0/nuoBQeensDa4sPeB1HJKwFYk6LnsBlwJNNWDbkpmn0l5nx8JVZdOkQzfS5q6isrvU6kkjYCsRNi5nA3c65Rh86C9VpGv2V2CGGP/5wOFv3HuY3/7vO6zgiYSsQhZcNzDWzbTRM6POEmV0agPW2KWeflsS/jj+NF5fvYF7+Tq/jiISlRmcta4xzrs9X35vZs8A859wJJ+MOZ7d/tz+fbS7j3tfWkNqpPSPTu3gdSSSsNOWxlDnA58AAMysys+vN7EYzu7Hl47Ut0ZERPD51BHExkVzx5GJueWElO8orvY4lEjY0TaMHDh+t5c8LCvifBQXU1Tt+fFY6t5zXl85xMV5HEwl5mqYxyHSIjeKO8/vzyV3juWxET/7y2VbOmfEJTy8s4GhtndfxRNosFZ6HUhLa8dCVWbw1/TsM692Z381fz/mPLmB+foleIirSAlR4QWBQagLPXTeKv103iriYSG5+YSWXP7mYFYWNPe8tIqdChRdEzunfjfnTv8PDV2RRvO8IVzz5OTc9v4JtepmoSED4/ViKBFZkhPGDM3tz0bBUnl64lT99uoUP1u/mR6PTmX5eP7p00I0NkebSEV6QiouJYvqEfnxy13iuHNmbvy3exrgZH/PUgi1U1ejGhkhzqPCCXHJ8O35/+VDeuX0c2eld+I+3NjDhD5/yZl6xXjklcopUeCGif0o8f/3nUcz+SQ6d2kdz29w8LnviM3ILyryOJhIyVHghZkzfJObdOpY/XDWM3QeP8sOnlnDDc8sp2HPI62giQU8jLULYkeo6nvlsK098vJmjtfVMy0lj+oR+dO0Y63U0Ec+cbKSFCq8N2FNxlMc+3MScpTuIi47kpnNP47oxfWgXHel1NJFWp6FlbVy3+Fh+d+lQ3r19HDmZXXn4nY2c98gnvL6qSDc2RI6hwmtD+iZ35Olrs5nz09F07RjLz15czcX/vYjFW/Z6HU0kKKjw2qCzTuvKmzeP4bEpw9l3uIar/yeXn/xtGZtLK7yOJuIpv6dpNLNLzCzfzPJ881WMDXxMOVUREcYlw3vy4Z3ncM+kgeQWlPP9mQu57/U17KnQPLkSnhq9aWFm44BDwHPOuSHH+bwjcNg558wsC3jJOTewsQ3rpkXrKj9czawPv+T5JYXERkVw0/jTuH5sJu1jdGND2ha/blo0Nk2jc+6Q+//W7ADoKnkQSuwQw68vPp33fjaOsf2SeOS9TZz7yCe8sqKIOt3YkDARkGt4ZnaZmW0A5gPXnWS5sJumMdhkduvIn6/J5uUbzyKlUzt+/vJqJj++iEVf6saGtH1Neg7PzDJomJznH05pv7XcOOAB59x3G1unTmm955xjXn4JD72zgaJ9Rxg/oBv3ThrEgO7xXkcTabZWew7Pd/qbaWZJgVyvtAwzY/KwHnx45zncd8EgVhbuY9JjC7j3tXxKD1Z5HU8k4PwuPDPra2bm+/4MIBbQiPYQEhsVyU/HZfLpXefyT2f34ZUVRYx/5BMe++BLKqtrvY4nEjBNuUs7BxgPJAG7gV8B0QDOuT+Z2d3Aj4Ea4Ahwl3NuUWMb1ilt8CosO8zD72xk/poSkuNjufN7/blyZG8iI8zraCKN0lhaaZYVheX8+/z1rNy+n4Hd47n3gkGc07+b17FETkpjaaVZRqYn8upNZ/PEtDOorK7j2meWcs1fcllfctDraCLNosKTkzIzLhiaygd3nMMvLxrMmuIDXDBrIXe9vJpdB3RjQ0KLTmnllByorOG/P9nMs59tIyICrh/bh2vPyiA5oZ3X0UQAXcOTFrCjvJKH393I31fvJCrCOH9wCtNy0jn7tK5E6OaGeEiFJy1m297DzFm6nZeW72BfZQ19kjpw9ag0rhzZS1NKiidUeNLiqmrqeGftLmbnFrJs2z5ioiK4cGgq03LSGJneBd+jmiItToUnrWrjrgpeyC3ktZXFVBytZWD3eKblpHHpiJ7Et4v2Op60cSo88URldS1/X72T55dsZ03xAeJiIrlkeA+m5aQzpGcnr+NJG6XCE8/lF+1n9pLtvLm6mKqaeob16sS0nHQmD+uhd/JJQKnwJGgcOFLDG6uKeX5JIV+WHiK+XRRXnNGLaTlp9EvRW1rEfyo8CTrOOZZt28fs3ELeXrOL6rp6RvVJZFpOGhOHdCc2Skd90jwqPAlqZYeO8sqKIl5Yup3CskoSO8RwVXYvpo1KJ61rnNfxJMSo8CQk1Nc7Ptuyl9lLtvP++t3U1TvG9e/GtJw0JgxMJipSIyGlcSo8CTm7DlTx4rIdzF22nZIDVaQkxDLlzDSmjOpNaqf2XseTIOZX4ZnZM8BFQOkJZi2bBtwNGFAB3OScW91YKBWeNEVtXT0fb9zD7NxCPt20BwMmDErhR6PT+U7fJA1jk39wssKLasKffxb4L+C5E3y+FTjHObfPzCYBTwE5zQkq8m1RkRGcPziF8wensL2skjnLtvPSsh28v243vRPbc/WodK7K7kVSx1ivo0oICPQkPl2Atc65no2tU0d40lzVtfW8+0XDMLYlBeVERxoTh6Tyo5w0RvVJ1DC2MOfvEd6puB54+yRBbgBuAEhLSwvwpiVcxERFMHlYDyYP68Hm0kO8kLudV1bs4O+rd9I3uSPTctK4fEQvOsVpGJt8U8CO8MzsXOAJYKxzrtFJfHSEJ4F0pLqOefk7mZ27nbwd+2kXHcHkrB5MG53OsF6ddNQXRlr8CM/MsoCngUlNKTuRQGsfE8lV2b25Krs3a4sP8MLS7byxqpiXVxRxeo8EpuWkc8nwHnSIDfRJjYQSv4/wzCwN+Aj4sXNucVM3rCM8aWkVVTW8kbeT2UsK2bCrgo6xUVw2oifTRqcxsHuC1/Gkhfj7WEpj0zQ+DVwBFPr+SO2JNnYsFZ60FuccK7fvZ3ZuIfPyS6iurWdkehem5aRxwdBU2kVrGFtbogePRXz2V1Y3DGPL3U7B3sN0jovmqpG9mDoqjcxuHb2OJwGgwhP5Fuccn28pY3budt79Yhe19Y5RfRKZnJXK94d0JzlekxKFKhWeyEmUVlTx8vIiXl9VzObSQ5hBTp9ELhyq8gtFKjyRJtq0u4L5+SXMX1Oi8gtRKjyRZvh2+UUYjFL5BT0VnoifNu2uYF5+CW8dp/wmDkmlW7zG8gYLFZ5IAJ2w/LJ6MPH07io/j6nwRFqAc45Nuw8xf43KL5io8ERa2LHlNz9/J1v2HCbCIKdPVy7ISlX5tSIVnkgrUvl5S4Un4hGVX+tT4YkEga/LL38n89eUqPxaiApPJMicrPwuzEpl4pDuem19M6nwRIKYc46Nuyt4K7+EeWtKKFD5+UWFJxIiTlR+ozO7csFQlV9TtPQ0jQOBvwJnAPc55x5pSigVnsjJqfyax9/CGwccAp47QeElA+nApcA+FZ5I4H1Vfl+N7VX5nZjfp7RNnMTn18AhFZ5Iy1L5nVzQFN63pmkcWVhYeKJFRaQJnHNs2FXBW2u+WX5ZvTqTk5nI6MyuZKd3Ib5d+ExZGTSFdywd4YkE1lfl9/baXSzevJfVRfupqXNEGAzt2YmczK6MzkwkOyORhDZcgK05EbeIeMTMGJSawKDUBO44vz9HqutYuX0fuQVlLCko59nPtvHUggIiDE7v0YmcPonkZHZlVEZi2ExarsITaaPax0Qypm8SY/omAVBV81UBlrOkoIznlhTy9KKtmMGg7gmMzuxKTmYiOX0S6RwX43H6lhGIaRq7A8uBBKCehju6g51zB0+2Xp3SinirqqaO1Tv2s6SgnNytZawo3MfR2nrMYEBKPKN9p8Cj+nQlsUPoFKAePBaRRh2trSO/6ABLtpSRu7Wc5YXlVNXUAw0F2HD013AUGMx3gVV4InLKqmvrWVPccAS4pKDhCLCyug6AvskdGX1MAQbT/B4qPBHxW01dPWuKD3x9DXD5tnIO+wows1uHhmuAfRoehUlJ8K4AVXgiEnC1dfWs3XmQ3IKGU+BlW8upOFoLQJ+kDl+XX05mIqmd2rdaLhWeiLS4unrHup0HWVJQRu7WhhKsqGoowPSucccUYFd6dm65AlThiUirq6t3rC85SO7WhlPgpVvLOXCkBoBeXdp/4xS4d2JcwLarwhMRz9XXN4wEyd1a9nUB7qtsKMCendt/4xQ4LTEOM2vWdlR4IhJ06usdX5Ye+v9T4IJyyg5XA5Daqd3XBThhUMopvfpeQ8tEJOhERBgDusczoHs8156dgXOOzb4CXLK1nEWby3gjbyezfxIXsLk+VHgiEhTMjH4p8fRLieeasxoKcMuew/TqErgbHCo8EQlKZkbf5I4BXWdEQNcmIhLEVHgiEjZUeCISNlR4IhI2VHgiEjY8e/DYzPYApzqLTxKwtwXieKGt7Etb2Q/QvgSrU92XdOdct+N94FnhNYeZLT/RE9Shpq3sS1vZD9C+BKtA7otOaUUkbKjwRCRshFrhPeV1gABqK/vSVvYDtC/BKmD7ElLX8ERE/BFqR3giIs0WEoVnZhPNbKOZbTaze7zO4w8ze8bMSs1srddZ/GFmvc3sYzNbZ2ZfmNltXmdqLjNrZ2ZLzWy1b19+43Umf5hZpJmtMrN5Xmfxh5ltM7M1ZpZnZgF5eWbQn9KaWSSwCTgfKAKWAVOdc+s8DdZMZjaOhsnKn3PODfE6T3OZWSqQ6pxbaWbxwArg0lD8/2INr9bt4Jw7ZGbRwCLgNufcEo+jNYuZ3QFkAwnOuYu8ztNcZrYNyHbOBex5wlA4whsFbHbOFTjnqoG5wCUeZ2o259wCoNzrHP5yzpU451b6vq8A1gM9vU3VPK7BId+P0b6v4D4SOAEz6wVcCDztdZZgFAqF1xPYcczPRYToX6y2yswygBFArrdJms93GpgHlALvO+dCdV9mAr8A6r0OEgAOeM/MVpjZDYFYYSgUngQxM+sIvArc7pw76HWe5nLO1TnnhgO9gFFmFnKXG8zsIqDUObfC6ywBMtY5dwYwCbjZdznIL6FQeMVA72N+7uX7nXjMd73rVWC2c+41r/MEgnNuP/AxMNHrLM0wBrjYd+1rLnCemT3vbaTmc84V+/5bCrxOw+Utv4RC4S0D+plZHzOLAaYA/+txprDnu9D/F2C9c+5Rr/P4w8y6mVln3/ftabhBtsHbVKfOOXevc66Xcy6Dhr8nHznnfuRxrGYxsw6+m2GYWQfge4DfTzYEfeE552qBW4B3abgw/pJz7gtvUzWfmc0BPgcGmFmRmV3vdaZmGgNcQ8NRRJ7v6wKvQzVTKvCxmeXT8A/s+865kH6kow1IARaZ2WpgKTDfOfeOvysN+sdSREQCJeiP8EREAkWFJyJhQ4UnImFDhSciYUOFJyJhQ4UnImFDhSciYUOFJyJh4/8AhUqGCfv0WrcAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 360x216 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAADSCAYAAAA/vMlrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcRUlEQVR4nO3deXxV9Z3/8deHsO+QhDWEsK8qyy2IWNeCuBTwp06VtuPS1k47VDv91RmtWhVtp9vP1l91ZmqtU+2itrhFbRUs0rogkLAoCWsCSkBCSCDsIctn/rgHes0ECeGGk3vv+/l45CFnu/dzWd5+v+ec+znm7oiIpLJWYRcgIhI2BaGIpDwFoYikPAWhiKQ8BaGIpDwFoYikPAWhiKQ8BaGIpDwFoSQdi9LfbWk0/WWRZmNmt5tZkZntM7NCM7syZttXzGxtzLYJwfoBZvacmZWZWbmZPRysv9fMfhtzfI6ZuZm1DpYXm9n3zOxt4CAw2MxujHmPYjP7ar36ZpnZKjPbG9Q5w8yuMbP8evt9y8xebL7fKQlb67ALkKRWBHwa2AFcA/zWzIYC5wL3ArOBPGAIUG1macDLwCLgi0AtEDmJ9/sicCmwHjBgBHAFUAycB/zZzJa7+wozmwQ8CVwN/AXoC3QBNgO/MLNR7r425nUfaMpvgCQGjQil2bj7H919u7vXufszwEZgEvBl4EfuvtyjNrn7B8G2fsBt7n7A3Q+7+1sn8Za/dvcCd69x92p3f8Xdi4L3+CuwgGgwA3wJeNzdFwb1bXP3de5eBTwDfAHAzMYAOUQDWpKUglCajZn9YzD13GNme4CxQAYwgOhosb4BwAfuXtPEt9xa7/0vNbN3zawieP/Lgvc/+l4N1QDwBDDHzIzoaPAPQUBKklIQSrMws4HAL4G5QLq7dwfWEJ2ybiU6Ha5vK5B99LxfPQeAjjHLfRrY51grJTNrBzwL/AToHbz/n4L3P/peDdWAu78LHCE6epwD/KbhTynJQkEozaUT0WAqAzCzG4mOCAEeA75tZhODK7xDg+BcBnwE/MDMOplZezObGhyzCjjPzLLNrBtwxwnevy3QLnj/GjO7FJges/1XwI1mdrGZtTKz/mY2Mmb7k8DDQPVJTs8lASkIpVm4eyHw/4AlQClwBvB2sO2PwPeA3wP7gBeAnu5eC3wWGAp8CJQAnwuOWUj03N17QD4nOGfn7vuAW4A/ALuJjuxyY7YvA24EfgpUAn8FBsa8xG+IBvdvkaRnaswq8r+ZWQdgJzDB3TeGXY80L40IRRr2NWC5QjA16D5CkXrMbAvRiyqzQy5FThNNjUUk5WlqLCIpT0EoIimvxZ0jzMjI8JycnLDLEJEkk5+fv8vdMxva1uKCMCcnh7y8vLDLEJEkY2YfHG+bpsYikvIUhCKS8hSEIpLyFIQikvJa3MUSEZHj2X3gCEs3l7OkqJxzh2UybXTvuLyuglBEWqzKQ9Us21zBkqJylhSXs27HXtyhQ5s0+nbvwDQUhCKSZPZX1bB8SwXvBsG3ZlsldQ7tWrdi4sAefOszw5kyJJ0zs7rTtnX8zuwpCEUkNIeO1JL/wW7eKdrFkuJy3iuppLbOaZNmjB/Qg7kXDWPK4HTGZ3enfZu0ZqtDQSgip83h6lpWfriHJcXlvFtUzsqtu6muddJaGWdmdeOfzh/MlMEZTBzYgw5tmy/46lMQikizOVJTx3sle46d48v/YDdVNXW0Mhjbvxs3TR3E2UPS+VROTzq3Cy+OFIQiEjc1tXWs2b43OtUtKidvy24OVdcCMKpvVz4/eSBThqQzaVBPunVoE3K1f6cgFJEmq61z1n6099iIb9nmCvZXRZ/GOqxXZ/4hksWUIelMHpROj05tQ672+BSEItJodXXOhp37osFXVM7SzRVUHqoGYHBGJ2aO68eUwemcPTidzC7tQq628RSEInJc7k5R2QGWBFd13y2uoOLAEQAG9OzAJWN6M2VIOlMGZ9CnW/uQq206BaGIHOPufFB+kCXF5cemu2X7qgDo2609FwzPjAbfkHSyenQMudr4URCKpLiS3QePhd6SonI+qjwMQGaXdkwZnB6M+NIZmN4RMwu52uahIBRJMTsqD7OkeNex8NtacQiAnp3acvbgnnw9CL8hmZ2TNvjqUxCKJLnq2jqWba5gQcEO3ty4i+JdBwDo2r41kwenc9PUQUwZks7wXl1o1So1gq8+BaFIEjpQVcNfN5SxoGAHi9btZO/hGtq1bsU5Q9K5blI2U4akM6pvV9JSNPjqUxCKJImyfVW8vraUhYWlvLVpF0dq6ujRsQ3Tx/Rh2ujefHpYBh3b6p98Q/S7IpLAisv2s7CwlAWFpaz4cDfukNWjA1+YPJDpY3oTGdiD1mnqv3wiCkKRBFJX57y3rZIFBTtYUFjKpp37ARjTryvfvHg408f0ZmSfLilzkSNeGhWEZjYDeAhIAx5z9x80sM8/APcCDqx29znB+uuBu4LdHnD3J+JQt0jKOFJTx5LichYU7OD1taWU7q0irZUxeVBPvjA5m8+M7p1U9/SF4YRBaGZpwCPANKAEWG5mue5eGLPPMOAOYKq77zazXsH6nsA9QIRoQOYHx+6O/0cRSR57D1ezeH30Ysdf15exr6qGjm3TOH94JtPH9ObCEb3o3rHlfnc30TRmRDgJ2OTuxQBm9jQwCyiM2ecrwCNHA87ddwbrLwEWuntFcOxCYAbwVHzKF0keOyoPszC42LGkaBfVtU5G57ZcfmZfpo3uzdShGc3anDSVNSYI+wNbY5ZLgMn19hkOYGZvE50+3+vurx7n2P5NrlYkiUS/x7uf1wqiFztWb90DQE56R26cOojpo3szPruHbnE5DeJ1saQ1MAy4AMgC/mZmZzT2YDO7GbgZIDs7O04libQ8tXXOqq27WRCE3+bg5uazBnTntktGMH10b4b2Sp1vdLQUjQnCbcCAmOWsYF2sEmCpu1cDm81sA9Fg3EY0HGOPXVz/Ddz9UeBRgEgk4o2sXSQhHK6u5Z2iXSwoKOX1taXs2n+E1q2MKUPSuencQUwb1TuhO7ckg8YE4XJgmJkNIhps1wJz6u3zAnAd8N9mlkF0qlwMFAHfN7MewX7TiV5UEUlqlQerWbQ+er5v8foyDh6ppXO71lwwIpPpY/pwwYhMurZvOR2aU90Jg9Dda8xsLvAa0fN/j7t7gZnNA/LcPTfYNt3MCoFa4DZ3Lwcws/uJhinAvKMXTkSSzfY9h4Kbm3ewtLiCmjqnV5d2XDm+P9NGR/v2tWutix0tkbm3rJloJBLxvLy8sMsQOSF3Z33pvuB83w7WbNsLwNBenZk+ujfTRvfmrKzuKdvIoKUxs3x3jzS0Td8sETkJtXVO3pYKFgQjv60VhzCDCdk9uP3SkUwb3ZshmZ3DLlNOkoJQ5AQOHanlzY1lLCgsZdG6nVQcOELbtFZMHZrO1y8YysWjetGriy52JDIFoUgDauuctzftYn5+CQsKd3C4uo4u7Vtz8cheTB/Th/OGZ4b6HF6JL/1JisQoKtvPs/klPLdiGzv2HqZbhzZcPTGLGWP6MnlwT9qok0tSUhBKyqs8VM0r733E/PytrPhwD2mtjPOHZ/Ldz47m4lG9dKU3BSgIJSXV1jlvBVPf1wp2cKSmjuG9O/Ody0Yye1x/enXVOb9UoiCUlLJp536eXVHCcytKKN1bRfeObbjuUwO4euIAxvbvqq+2pSgFoSS9ykPVvPzedubnl7AymPpeMDyTez+bxUWa+goKQklStXXOmxvLgqu+pRypqWNE7y7cedkoZo3vp9td5GMUhJJUNu3cx/z8bTy/8u9T3zmTsrl6YhZj+mnqKw1TEErCqzxYzUvB1HfV1ujU98IRmdw3M4sLR2rqKyemIJSEdLyp712Xj2LWuP5kdmkXdomSQBSEklA27dzHH/NLeH7FNnbu09RX4kNBKC1e5cFqcoOp7+qYqe/VEzX1lfhQEEqLVFNbx5vBDc8Lg6nvyD6a+krzUBBKi7KxdB/zV/x96ttDU185DRSEErqGp769uHpiFheN7EXb1mp0IM1LQSihqKmt482NMVPfWk19JTwKQjmtNpTui7a5WrmNsn1V9OzUls+fnc1VEzT1lfAoCKXZ7Tl4hJdWB1PfkkpatzIuHBmd+l44QlNfCZ+CUJrF8aa+d18xmlnj+pHRWVNfaTkUhBJX7s6Lq7bz739eS+nev099o1d9u4VdnkiDGhWEZjYDeIjoc40fc/cf1Nt+A/Bjog+AB3jY3R8LttUC7wfrP3T3mXGoW1qgorL93P3CGt4pKuesAd2ZN2uspr6SEE4YhGaWBjwCTANKgOVmluvuhfV2fcbd5zbwEofcfdyplyot1eHqWv5jcRH/tbiIdm1a8cDsscyZlK3n+UrCaMyIcBKwyd2LAczsaWAWUD8IJQW9ubGMu19Yw5byg8we1487Lx+tW18k4TQmCPsDW2OWS4DJDex3lZmdB2wA/sXdjx7T3szygBrgB+7+Qv0Dzexm4GaA7OzskyhfwrJz32EeeHktuau3MyijE7/78mSmDs0IuyyRJonXxZKXgKfcvcrMvgo8AVwUbBvo7tvMbDCwyMzed/ei2IPd/VHgUYBIJOJxqkmaQW2d8/ulH/Cj19ZTVVPHv3xmOF89fzDt26jxgSSuxgThNmBAzHIWf78oAoC7l8csPgb8KGbbtuC/xWa2GBgPfCwIJTGs2VbJnc+/z+qSSs4dmsH9s8cyKKNT2GWJnLLGBOFyYJiZDSIagNcCc2J3MLO+7v5RsDgTWBus7wEcDEaKGcBUYkJSEsP+qhoeXLCBX7+zmZ6d2vHQteOYeVY/fQtEksYJg9Dda8xsLvAa0dtnHnf3AjObB+S5ey5wi5nNJHoesAK4ITh8FPALM6sDWhE9R6iLLAnC3Xl1zQ7ue6mQ0n2H+cLkgXz7khF069Am7NJE4srcW9YpuUgk4nl5eWGXkfK2Vhzkuy+u4Y31ZYzu25XvXTmW8dk9wi5LpMnMLN/dIw1t0zdL5GOO1NTxyzeL+fmijaSZcfcVo7l+ykBap+mmaEleCkI5ZmlxOXe9sIaNO/czY0wf7pk5mr7dOoRdlkizUxAKFQeO8O9/Wssf80vo370Dv7o+wsWjeoddlshpoyBMYXV1zvz8Er7/57XsP1zD1y4Ywi0XDaNDW90TKKlFQZiiNpTu467n17BsSwWfyunB9648g+G9u4RdlkgoFIQp5tCRWv7/oo388m/FdG7fmh9ddSZXT8xSgwRJaQrCFLJoXSnffbGAkt2HuGZiFndcNoqendqGXZZI6BSEKeCjykPcl1vIqwU7GNqrM8/cfDaTB6eHXZZIi6EgTGI1tXU8seQDHlywnpo657ZLRvCVTw9Wo1SRehSESWrV1j1857n3KfxoLxeMyGTezLFkp3cMuyyRFklBmGQqD1Xz49fW8bulH9KrSzv+8/MTmDG2jxokiHwCBWGScHdyV2/n/pfXUnGgihvOyeFb04bTpb0aJIiciIIwCWzedYC7X1jDW5t2cWZWN35946cY219PjBNpLAVhAquqqeU/FxfxH4uLaJfWivtnjWHO5IGk6Z5AkZOiIExQb2/axd0vrKF41wE+e1Y/7r58FL26tg+7LJGEpCBMMGX7qnjglUJeXLWdgekdefKmSZw3PDPsskQSmoIwQdTVOb9f9iE/fHUdVdV13HLxML5+wRA9NEkkDhSECaBgeyV3Pr+GVVv3cM6QdO6fPZYhmZ3DLkskaSgIW7D9VTX8dOEG/vvtzfTo2Jaffu4sZo/rr3sCReJMQdgCuTuvFZRy30sFfFR5mDmTs/m3S0bSraPuCRRpDgrCFqZk90HuebGAv6zbycg+XXh4zgQmDtRDk0SaU6O+fW9mM8xsvZltMrPbG9h+g5mVmdmq4OfLMduuN7ONwc/18Sw+2Ty5ZAvTHvwbS4rLufOyUbz8jXMVgiKnwQlHhGaWBjwCTANKgOVmltvA84mfcfe59Y7tCdwDRAAH8oNjd8el+iTyemG0V+D5wzP5/v85g/7d9dAkkdOlMSPCScAmdy929yPA08CsRr7+JcBCd68Iwm8hMKNppSavHZWHuW3+akb37covvjhRIShymjUmCPsDW2OWS4J19V1lZu+Z2XwzG3CSx6as2jrn1qdXUlVTx8/njNd9gSIhiFeHzpeAHHc/k+io74mTOdjMbjazPDPLKysri1NJieHhRZtYurmCebN0b6BIWBoThNuAATHLWcG6Y9y93N2rgsXHgImNPTY4/lF3j7h7JDMzdb4utrS4nIf+soErx/fnqgkaKIuEpTFBuBwYZmaDzKwtcC2QG7uDmfWNWZwJrA1+/Row3cx6mFkPYHqwLuXtPnCEW59eRXbPjtw/e6xukhYJ0QmvGrt7jZnNJRpgacDj7l5gZvOAPHfPBW4xs5lADVAB3BAcW2Fm9xMNU4B57l7RDJ8jobg7t81/j/IDVTz3tal0bqfbOUXCZO4edg0fE4lEPC8vL+wymtWv397MvS8VcvcVo/nSuYPCLkckJZhZvrtHGtqmx5mdZmu2VfL9P63j4pG9uGlqTtjliAgKwtPqQFUNtzy1kh6d2vDja87SeUGRFkInp06j775YwObyA/z+y2fTs1PbsMsRkYBGhKfJ8ytLeHZFCd+4aBhThqSHXY6IxFAQngabdx3grufXMCmnJ7dcNDTsckSkHgVhM6uqqeUbT62gdVorfnbtOFqn6bdcpKXROcJm9sM/r2fNtr08+sWJ9FMzBZEWScOTZvSXtaU8/vZmbjgnh+lj+oRdjogch4KwmeyoPMy3/7iaUX27cvulI8MuR0Q+gYKwGcS21npYrbVEWjydI2wGR1tr/eSas9RaSyQBaEQYZ8s2V/DQXzYwe1w/tdYSSRAKwjiKttZaSXbPjjxw5Rn6Cp1IgtDUOE6OttbatV+ttUQSjUaEcfLEO1t4fW0pt186ijOyuoVdjoicBAVhHBRsj7bWukittUQSkoLwFB2oquEbvw9aa119ps4LiiQgncg6RbGttdI7twu7HBFpAo0IT8Gx1loXDlVrLZEEpiBsoqOttT6V04NbLh4WdjkicgoUhE0Q21rroWvHq7WWSILTOcImUGstkeTSqKGMmc0ws/VmtsnMbv+E/a4yMzezSLCcY2aHzGxV8PNf8So8LEdba10/ZaBaa4kkiROOCM0sDXgEmAaUAMvNLNfdC+vt1wW4FVha7yWK3H1cnOoNVWxrrTsuGxV2OSISJ40ZEU4CNrl7sbsfAZ4GZjWw3/3AD4HDcayvxVBrLZHk1Zgg7A9sjVkuCdYdY2YTgAHu/koDxw8ys5Vm9lcz+3TTSw3XI29EW2vdN3OMWmuJJJlTvlhiZq2AB4EbGtj8EZDt7uVmNhF4wczGuPveeq9xM3AzQHZ29qmWFHfLNlfws9ejrbWunpgVdjkiEmeNGRFuAwbELGcF647qAowFFpvZFuBsINfMIu5e5e7lAO6eDxQBw+u/gbs/6u4Rd49kZmY27ZM0E7XWEkl+jQnC5cAwMxtkZm2Ba4HcoxvdvdLdM9w9x91zgHeBme6eZ2aZwcUWzGwwMAwojvunaCbuzr8+G22t9fPrJqi1lkiSOuG/bHevMbO5wGtAGvC4uxeY2Twgz91zP+Hw84B5ZlYN1AH/5O4V8Sj8dHhyyQcsLCzlrsvVWkskmZm7h13Dx0QiEc/Lywu7DAq2V3LlI+9w7rAMfnV9RFNikQRnZvnuHmlom74b1oCjrbW6d1RrLZFUoJNeDbgnN9pa63dfnqzWWiIpQCPCep5fWcL8/GhrrXOGZIRdjoicBgrCGGqtJZKaFIQBtdYSSV06Rxj40avR1lq/UGstkZSjYQ/R1lq/eivaWusStdYSSTkpH4RqrSUiKR2EtXXON59ZyeHqOn5+nVpriaSqlD5H+Mgbm3i3uIIfX30mQ3uptZZIqkrZEeHR1lqz1FpLJOWlZBDuORhtrTWgZ0cemD1WX6ETSXEpNzV2d26bH22t9ezXzqFL+zZhlyQiIUu5EeHR1lr/NmMkZ2Z1D7scEWkBUioIC7ZX8r1X1nLhiEy+dO6gsMsRkRYiZYLwQFUN33gq2lrrJ9ecpfOCInJMypwjvCe3gM271FpLRP63lBgRvrByG/PzS5ir1loi0oCkD8Ituw5w5/PvExnYg1vVWktEGpDUQVhVU8vco621rlNrLRFpWFKfI4xtrdVfrbVE5DiSdoi0aF20tdY/qrWWiJxAo4LQzGaY2Xoz22Rmt3/CfleZmZtZJGbdHcFx683skngUfSLR1lrvMbJPF76j1loicgInnBqbWRrwCDANKAGWm1muuxfW268LcCuwNGbdaOBaYAzQD3jdzIa7e238PsLHHW2tdehILQ/PmaDWWiJyQo0ZEU4CNrl7sbsfAZ4GZjWw3/3AD4HDMetmAU+7e5W7bwY2Ba/XbI621rpv1hi11hKRRmlMEPYHtsYslwTrjjGzCcAAd3/lZI8Njr/ZzPLMLK+srKxRhTcktrXWNWqtJSKNdMoXS8ysFfAg8H+b+hru/qi7R9w9kpmZ2aTXUGstEWmqxtw+sw0YELOcFaw7qgswFlgchE8fINfMZjbi2LhQay0RORWNGREuB4aZ2SAza0v04kfu0Y3uXunuGe6e4+45wLvATHfPC/a71szamdkgYBiwLN4f4jfvqrWWiDTdCUeE7l5jZnOB14A04HF3LzCzeUCeu+d+wrEFZvYHoBCoAf453leMC7ZX8sDL0dZaN01Vay0ROXnm7mHX8DGRSMTz8vIavf/XfptP/ge7+fOtn1ZXGRE5LjPLd/dIQ9sS/it2P/3cOD6sOKgQFJEmS/iv2LVvk8bw3l3CLkNEEljCB6GIyKlSEIpIylMQikjKUxCKSMpTEIpIymtx9xGaWRnwwUkelgHsaoZywqDP0vIky+eA1P4sA929wWYGLS4Im8LM8o53o2Si0WdpeZLlc4A+y/FoaiwiKU9BKCIpL1mC8NGwC4gjfZaWJ1k+B+izNCgpzhGKiJyKZBkRiog0WcIHYWMfNdrSmdnjZrbTzNaEXcupMLMBZvaGmRWaWYGZ3Rp2TU1lZu3NbJmZrQ4+y31h13QqzCzNzFaa2cth13IqzGyLmb1vZqvMrPE9+z7pNRN5ahw8anQDMY8aBa6r/6jRRGBm5wH7gSfdfWzY9TSVmfUF+rr7iuARr/nA7AT9MzGgk7vvN7M2wFvAre7+bsilNYmZfQuIAF3d/Yqw62kqM9sCRNw9bvdDJvqIsLGPGm3x3P1vQEXYdZwqd//I3VcEv94HrKWBJxcmAo/aHyy2CX4ScuRgZlnA5cBjYdfSEiV6EDbqcaESDjPLAcYDS8OtpOmC6eQqYCew0N0T9bP8DPhXoC7sQuLAgQVmlm9mN8fjBRM9CKWFMrPOwLPAN919b9j1NJW717r7OKJPYJxkZgl32sLMrgB2unt+2LXEybnuPgG4FPjn4LTSKUn0IDwtjwuVkxOcT3sW+J27Pxd2PfHg7nuAN4AZYdfSBFOBmcG5taeBi8zst+GW1HTuvi34707geaKnyE5JogfhJz5qVE6/4ALDr4C17v5g2PWcCjPLNLPuwa87EL0oty7cqk6eu9/h7lnB43avBRa5+xdCLqtJzKxTcBEOM+sETAdO+U6LhA5Cd68Bjj5qdC3wB3cvCLeqpjGzp4AlwAgzKzGzL4VdUxNNBb5IdNSxKvi5LOyimqgv8IaZvUf0f7oL3T2hbz1JAr2Bt8xsNdFnpL/i7q+e6osm9O0zIiLxkNAjQhGReFAQikjKUxCKSMpTEIpIylMQikjKUxCKSMpTEIpIylMQikjK+x97q4brS44bmAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Inference\n",
        "\n",
        "With enough training epochs, the test accuracy should exceed 96.53%.\n",
        "You can compare your result with the state-of-the art here."
      ],
      "metadata": {
        "id": "qHqH_DkAMde4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "x_test_reshaped = numpy.expand_dims(x_test, -1)\n",
        "scores = cnn_model.evaluate(x_test, y_test, verbose=2)\n",
        "print(\"%s: %.2f%%\" % (cnn_model.metrics_names[1], scores[1]*100))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JcOI6OZUMxAO",
        "outputId": "404b7ea9-8c75-4360-97f3-93b0aaae84e4"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "313/313 - 14s - loss: 0.9610 - accuracy: 0.6625 - 14s/epoch - 45ms/step\n",
            "accuracy: 66.25%\n",
            "CPU times: user 15.5 s, sys: 186 ms, total: 15.7 s\n",
            "Wall time: 15.4 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can also again check the confusion matrix"
      ],
      "metadata": {
        "id": "y0W4m0CxNVMV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "print('Confusion matrix (rows: true classes; columns: predicted classes):'); print()\n",
        "predictions = cnn_model.predict(x_test)\n",
        "cm=confusion_matrix(y_test, numpy.argmax(predictions, axis=1), labels=list(range(10)))\n",
        "print(cm); print()\n",
        "\n",
        "print('Classification accuracy for each class:'); print()\n",
        "for i,j in enumerate(cm.diagonal()/cm.sum(axis=1)): print(\"%d: %.4f\" % (i,j))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_uwyvZTGNWOv",
        "outputId": "39ec4904-4eec-4004-aadb-db982249fc67"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Confusion matrix (rows: true classes; columns: predicted classes):\n",
            "\n",
            "[[684  16  85  22  21   4  18   9  97  44]\n",
            " [ 25 732  11  11   4   3  19   8  42 145]\n",
            " [ 61   4 558  54  88  82  93  32  12  16]\n",
            " [ 18   8 101 421  76 166 130  42  15  23]\n",
            " [ 28   2 119  53 581  23  86  90  15   3]\n",
            " [ 10   2 106 166  49 560  38  56   6   7]\n",
            " [  4   3  56  41  42  16 821   8   3   6]\n",
            " [ 16   2  48  40  67  69   8 732   1  17]\n",
            " [ 78  56  17  11   6  10  16   2 764  40]\n",
            " [ 38  77  11  12  10   7  13  24  36 772]]\n",
            "\n",
            "Classification accuracy for each class:\n",
            "\n",
            "0: 0.6840\n",
            "1: 0.7320\n",
            "2: 0.5580\n",
            "3: 0.4210\n",
            "4: 0.5810\n",
            "5: 0.5600\n",
            "6: 0.8210\n",
            "7: 0.7320\n",
            "8: 0.7640\n",
            "9: 0.7720\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "More verbose training\n",
        "This approach explicitly handles the looping over data. It will be helpful in future weeks for diving in and optimizing"
      ],
      "metadata": {
        "id": "GwoyjzbsNlE7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_loss(y_true, y_pred):\n",
        "    # if labels are integers, use sparse categorical crossentropy\n",
        "    # network's final layer is softmax, so from_logtis=False\n",
        "    scce = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)\n",
        "    # if labels are one-hot encoded, use standard crossentropy\n",
        "\n",
        "    return scce(y_true, y_pred)  "
      ],
      "metadata": {
        "id": "Ckcl3Y9fNmza"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def forward_pass(model, batch_data, y_true):\n",
        "    y_pred = model(batch_data)\n",
        "    loss = compute_loss(y_true, y_pred)\n",
        "    return loss"
      ],
      "metadata": {
        "id": "Egz_bUK3Nv3f"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Here is a function that will manage the training loop for us:\n",
        "\n",
        "def train_loop(batch_size, n_training_epochs, model, opt):\n",
        "    \n",
        "    @tf.function()\n",
        "    def train_iteration(data, y_true, model, opt):\n",
        "        with tf.GradientTape() as tape:\n",
        "            loss = forward_pass(model, data, y_true)\n",
        "\n",
        "        trainable_vars = model.trainable_variables\n",
        "\n",
        "        # Apply the update to the network (one at a time):\n",
        "        grads = tape.gradient(loss, trainable_vars)\n",
        "\n",
        "        opt.apply_gradients(zip(grads, trainable_vars))\n",
        "        return loss\n",
        "\n",
        "    for i_epoch in range(n_training_epochs):\n",
        "        print(\"beginning epoch %d\" % i_epoch)\n",
        "        start = time.time()\n",
        "\n",
        "        epoch_steps = int(50000/batch_size)\n",
        "        dataset.shuffle(50000) # Shuffle the whole dataset in memory\n",
        "        batches = dataset.batch(batch_size=batch_size, drop_remainder=True)\n",
        "        \n",
        "        for i_batch, (batch_data, y_true) in enumerate(batches):\n",
        "            batch_data = tf.reshape(batch_data, [-1, 32, 32, 3])\n",
        "            loss = train_iteration(batch_data, y_true, model, opt)\n",
        "            \n",
        "        end = time.time()\n",
        "        print(\"took %1.1f seconds for epoch #%d\" % (end-start, i_epoch))"
      ],
      "metadata": {
        "id": "3VOO6BIDN2h0"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_network(_batch_size, _n_training_epochs, _lr):\n",
        "\n",
        "    cifar_model = CIFAR10Classifier()\n",
        "\n",
        "    opt = tf.keras.optimizers.Adam(_lr)\n",
        "\n",
        "    train_loop(_batch_size, _n_training_epochs, cifar_model, opt)"
      ],
      "metadata": {
        "id": "MReHhKgKOIXA"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
        "dataset.shuffle(50000)\n",
        "\n",
        "batch_size = 64\n",
        "epochs = 6\n",
        "lr = .05\n",
        "train_network(batch_size, epochs, lr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x6fmD1HHOLf7",
        "outputId": "922d1788-155b-4d0b-9577-b9b454741a1d"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "beginning epoch 0\n",
            "took 161.6 seconds for epoch #0\n",
            "beginning epoch 1\n",
            "took 169.7 seconds for epoch #1\n",
            "beginning epoch 2\n",
            "took 153.8 seconds for epoch #2\n",
            "beginning epoch 3\n",
            "took 156.9 seconds for epoch #3\n",
            "beginning epoch 4\n",
            "took 153.1 seconds for epoch #4\n",
            "beginning epoch 5\n",
            "took 153.0 seconds for epoch #5\n"
          ]
        }
      ]
    }
  ]
}