2022_12_07_Zahmeeth_Sakkaff_Session2_02_full_net.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/argonne-lcf/ai-science-training-series.git\n",
        "%cd ai-science-training-series/02_neural_networks_python/"
      ],
      "metadata": {
        "id": "yUychJPgKbfp",
        "outputId": "1c2e1d4d-fb06-43a8-89be-ec4dd33d792e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "id": "yUychJPgKbfp",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'ai-science-training-series'...\n",
            "remote: Enumerating objects: 1938, done.\u001b[K\n",
            "remote: Counting objects: 100% (1938/1938), done.\u001b[K\n",
            "remote: Compressing objects: 100% (991/991), done.\u001b[K\n",
            "remote: Total 1938 (delta 934), reused 1917 (delta 920), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (1938/1938), 210.12 MiB | 22.15 MiB/s, done.\n",
            "Resolving deltas: 100% (934/934), done.\n",
            "Checking out files: 100% (251/251), done.\n",
            "/content/ai-science-training-series/02_neural_networks_python\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "e19878bb",
      "metadata": {
        "id": "e19878bb"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "662a93d1",
      "metadata": {
        "id": "662a93d1"
      },
      "outputs": [],
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "da412dba",
      "metadata": {
        "id": "da412dba"
      },
      "outputs": [],
      "source": [
        "# repeating the data prep from the previous notebook\n",
        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
        "x_train = x_train.astype(numpy.float32)\n",
        "x_test  = x_test.astype(numpy.float32)\n",
        "\n",
        "x_train /= 255.\n",
        "x_test  /= 255.\n",
        "\n",
        "print(x_train.shape)\n",
        "x_train = x_train.reshape(x_train.shape[0], numpy.prod(x_train[0,:,:].shape))\n",
        "x_test = x_test.reshape(x_test.shape[0], numpy.prod(x_test[0,:,:].shape))\n",
        "\n",
        "print(x_train.shape)\n",
        "y_train = y_train.astype(numpy.int32)\n",
        "y_test  = y_test.astype(numpy.int32)\n",
        "\n",
        "print()\n",
        "print('MNIST data loaded: train:',len(x_train),'test:',len(x_test))\n",
        "print('X_train:', x_train.shape)\n",
        "print('y_train:', y_train.shape)\n",
        "\n",
        "# one-hot encoding:\n",
        "nb_classes = 10\n",
        "y_train_onehot = tf.keras.utils.to_categorical(y_train, nb_classes)\n",
        "y_test_onehot = tf.keras.utils.to_categorical(y_test, nb_classes)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "302994b1",
      "metadata": {
        "id": "302994b1"
      },
      "outputs": [],
      "source": [
        "# Here we import an implementation of a two-layer neural network \n",
        "# this code is based on pieces of the first assignment from Stanford's CSE231n course, \n",
        "# hosted at https://github.com/cs231n/cs231n.github.io with the MIT license\n",
        "from fc_net import TwoLayerNet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "4e00e3de",
      "metadata": {
        "id": "4e00e3de"
      },
      "outputs": [],
      "source": [
        "num_features = x_train.shape[1] # this is the number of pixels\n",
        "# The weights are initialized from a normal distribution with standard deviation weight_scale\n",
        "# model = TwoLayerNet(input_dim=num_features, hidden_dim=300, num_classes=nb_classes, weight_scale=.01) (Bethany Lush default parameter)\n",
        "model = TwoLayerNet(input_dim=num_features, hidden_dim=300, num_classes=nb_classes, weight_scale=.1) #Zahmeeth Sakkaff increased the weight_scale"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "32f7f1aa",
      "metadata": {
        "id": "32f7f1aa"
      },
      "outputs": [],
      "source": [
        "# here you can take a look if you want at the initial loss from an untrained network\n",
        "loss, gradients = model.loss(x_train, y_train_onehot)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "c43e3aa5",
      "metadata": {
        "id": "c43e3aa5"
      },
      "outputs": [],
      "source": [
        "# a simple implementation of stochastic gradient descent\n",
        "def sgd(model, gradients, learning_rate):\n",
        "    for p, w in model.params.items():\n",
        "        dw = gradients[p]\n",
        "        new_weights = w - learning_rate * dw\n",
        "        model.params[p] = new_weights\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "id": "c8316228",
      "metadata": {
        "id": "c8316228"
      },
      "outputs": [],
      "source": [
        "# one training step\n",
        "def learn(model, x_train, y_train_onehot, learning_rate):\n",
        "    loss, gradients = model.loss(x_train, y_train_onehot)\n",
        "    model = sgd(model, gradients, learning_rate)\n",
        "    return loss, model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "81886e8c",
      "metadata": {
        "id": "81886e8c"
      },
      "outputs": [],
      "source": [
        "def accuracy(model, x, true_values):\n",
        "    scores = model.loss(x)\n",
        "    predictions = numpy.argmax(scores, axis=1)\n",
        "    N = predictions.shape[0]\n",
        "    acc = (true_values == predictions).sum() / N\n",
        "    return acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "id": "49754891",
      "metadata": {
        "id": "49754891",
        "outputId": "4efbae17-d2c6-4238-b3da-5100f6865082",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch 0, loss 0.40031, accuracy 0.86\n",
            "epoch 1, loss 0.42735, accuracy 0.89\n",
            "epoch 2, loss 0.39870, accuracy 0.90\n",
            "epoch 3, loss 0.31596, accuracy 0.91\n",
            "epoch 4, loss 0.28716, accuracy 0.92\n",
            "epoch 5, loss 0.31998, accuracy 0.92\n",
            "epoch 6, loss 0.23625, accuracy 0.92\n",
            "epoch 7, loss 0.24337, accuracy 0.93\n",
            "epoch 8, loss 0.25318, accuracy 0.93\n",
            "epoch 9, loss 0.25249, accuracy 0.93\n",
            "epoch 10, loss 0.25866, accuracy 0.94\n",
            "epoch 11, loss 0.23246, accuracy 0.94\n",
            "epoch 12, loss 0.20730, accuracy 0.94\n",
            "epoch 13, loss 0.17896, accuracy 0.94\n",
            "epoch 14, loss 0.20556, accuracy 0.94\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f6a64a3f350>]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU5d3/8fd3JpMFCAlL2EF2EVFQUwWUioCKVGs3W+1iF6s/bWu1tbVq1Vq32tZa26dVH2tbtVr3pdYHF1BwFwn7DmEzYQ1bFrIn9++PmUxmkskCJBnO5PO6rlzOOefO5DuH8TNn7nOf+5hzDhER8T5fvAsQEZG2oUAXEUkQCnQRkQShQBcRSRAKdBGRBKFAFxFJEEktNTCzVOBdICXU/nnn3K8atEkBHgdOAfYCX3PObWnueXv37u2GDh16eFWLiHRSixYt2uOcy4q1rcVAByqAac65EjMLAO+b2WvOuY8j2lwG7HfOjTSzi4HfAl9r7kmHDh1KTk5OK1+CiIgAmNnWpra12OXigkpCi4HQT8OrkS4EHgs9fh6YbmZ2GLWKiMhhalUfupn5zWwpsBuY45xb0KDJQCAPwDlXDRQCvdqyUBERaV6rAt05V+OcmwAMAk41s3GH88fM7AozyzGznIKCgsN5ChERacIhjXJxzh0A5gEzG2zaBgwGMLMkIIPgydGGv/+wcy7bOZedlRWzT19ERA5Ti4FuZllmlhl6nAacDaxt0OwV4Nuhx18B3naa9UtEpEO1ZpRLf+AxM/MT/AB41jn3qpndDuQ4514B/g78y8xygX3Axe1WsYiIxNRioDvnlgMnxVh/a8TjcuCiti1NREQOheeuFF23s5j73lzHnpKKeJciInJU8Vyg5+4u4c9v57LvYGW8SxEROap4LtB9ocuVanXOVUQkiucCve4C1JpaBbqISCTPBbo/dIiuA3QRkWieC3R1uYiIxObBQA8munpcRESieS7Q6+ZwVB+6iEg0zwV6fR+6Al1EJJLnAl1dLiIisXku0NXlIiISm+cC3W/qchERicVzge7zqctFRCQW7wW6xqGLiMTkuUAPX/qvQBcRieK5QFcfuohIbJ4L9PCwxdo4FyIicpTxXKCHhy3qCF1EJIrnAl1XioqIxOa5QNeVoiIisXkw0IP/1bBFEZFongt03bFIRCQ2zwW67lgkIhKb5wJdXS4iIrF5MNDV5SIiEov3Al1dLiIiMXkv0NXlIiISU4uBbmaDzWyema02s1Vmdk2MNlPNrNDMloZ+bm2fcjUOXUSkKUmtaFMNXOecW2xm6cAiM5vjnFvdoN17zrnz277EaLr0X0QkthaP0J1zO5xzi0OPi4E1wMD2Lqwpmm1RRCS2Q+pDN7OhwEnAghibJ5nZMjN7zcyOb4PaYqqfbVGBLiISqTVdLgCYWTfgBeBa51xRg82LgWOccyVmNgt4GRgV4zmuAK4AGDJkyGEVHB62qDwXEYnSqiN0MwsQDPMnnXMvNtzunCtyzpWEHs8GAmbWO0a7h51z2c657KysrMMr2Bd+rsP6fRGRRNWaUS4G/B1Y45y7r4k2/ULtMLNTQ8+7ty0LrVM/ykWBLiISqTVdLqcD3wJWmNnS0LqbgCEAzrmHgK8AV5lZNVAGXOza6RBawxZFRGJrMdCdc+8D1kKbvwB/aauimhMetqhEFxGJ4rkrRXXHIhGR2DwX6OpyERGJzYOBHvyvulxERKJ5LtDNDDONchERachzgQ4Q8Pmo0pVFIiJRPBnoSX6juqY23mWIiBxVvBnoPqNafegiIlE8GejJST6qdIQuIhLFk4Ge5FOgi4g05M1A9xvVOikqIhLFk4Ee8PuoUh+6iEgUjwa6UVWtLhcRkUieDPQkn4/qWgW6iEgkTwZ6wG+6sEhEpAGPBrpGuYiINOTJQNcoFxGRxjwZ6MFRLjpCFxGJ5MlAT/LpCF1EpCFPBrr60EVEGvNmoCf5qFSgi4hE8WSgpyb5qahSoIuIRPJkoHdN8XOwsjreZYiIHFU8GehpyX5KK2viXYaIyFHFk4HeJZBEZXWt7lokIhLBk4HeNcUPQGmVjtJFROp4MtDTkoOBXqZuFxGRME8GetfkJAAOVujEqIhInRYD3cwGm9k8M1ttZqvM7JoYbczM/mxmuWa23MxObp9yg1IDoSN0dbmIiIQltaJNNXCdc26xmaUDi8xsjnNudUSb84BRoZ/TgAdD/20XAb8FC9Pl/yIiYS0eoTvndjjnFoceFwNrgIENml0IPO6CPgYyzax/m1cbkuQPlq2bXIiI1DukPnQzGwqcBCxosGkgkBexnE/j0G8zAV/wCF03uRARqdfqQDezbsALwLXOuaLD+WNmdoWZ5ZhZTkFBweE8BRBxhK5AFxEJa1Wgm1mAYJg/6Zx7MUaTbcDgiOVBoXVRnHMPO+eynXPZWVlZh1MvELzBBaA50UVEIrRmlIsBfwfWOOfua6LZK8ClodEuE4FC59yONqwzSsCnI3QRkYZaM8rldOBbwAozWxpadxMwBMA59xAwG5gF5AKlwHfbvtR6SeFRLjpCFxGp02KgO+feB6yFNg74YVsV1ZJAuMtFR+giInU8eaVoUrjLRUfoIiJ1PBnofp8uLBIRaciTgR4IDVvUKBcRkXqeDPQkXfovItKIJwO9bthilfrQRUTCPBno4SN0jXIREQnzdKDXKNBFRMI8GejqchERacyTge7zGT5ToIuIRPJkoEPwrkWV1Qp0EZE6ng503YJORKSeZwM9LeCnrFJH6CIidTwb6KkBH+XVOkIXEanj4UD3U16pQBcRqePZQE9TH7qISBTvBnqyn3IFuohImGcDPSXJT1mVToqKiNTxbKDrCF1EJJp3Az3go0wnRUVEwjwb6KkBv4YtiohE8GygBy8sUqCLiNTxbKCnBvxUVNdSqyl0RUQADwd6WrIfgApN0CUiAng40FOTgqXvKCyLcyUiIkcHzwZ6IBTo33hkQZwrERE5Ong20PeVVAKwo7A8zpWIiBwdPBvoXVOSgOBoFxERaUWgm9k/zGy3ma1sYvtUMys0s6Whn1vbvszGLp10DAAzxvbtiD8nInLUS2pFm0eBvwCPN9PmPefc+W1SUSsl+X2M6Zeuy/9FREJaPEJ3zr0L7OuAWg5ZakDzuYiI1GmrPvRJZrbMzF4zs+Pb6DlblBrwUaEZF0VEgLYJ9MXAMc658cD/AC831dDMrjCzHDPLKSgoOOI/rBtFi4jUO+JAd84VOedKQo9nAwEz691E24edc9nOueysrKwj/dOkJqnLRUSkzhEHupn1MzMLPT419Jx7j/R5WyMtWTMuiojUaXGUi5k9BUwFeptZPvArIADgnHsI+ApwlZlVA2XAxc65DpkxKzXgo1x96CIiQCsC3Tl3SQvb/0JwWGOHS1GXi4hImGevFIXQFLo6QhcRATwf6D4qa2qp0ZzoIiJeD/TgPC7qdhER8XigpynQRUTCPB3oqYFg+eW6a5GIiNcDXUfoIiJ1PB3oKUnBQC+rVKCLiHg60Ou6XCp0taiIiLcDvf6kqPrQRUQ8HejqQxcRqZcgga4jdBERjwd6aNiijtBFRLwe6KFRLgp0ERGPB3po2OLuovI4VyIiEn/eDvTkYPl/fjtXE3SJSKfn6UBP9teXP+0P8+NXiIjIUcDTgR668x0AW/eWxrESEZH483SgA9xx4fHxLkFE5Kjg+UCfMbZv+HGlZl0UkU7M84HeLaX+tqilldVxrEREJL48H+hdkyMDXePRRaTz8nyg+3z1J0YV6CLSmXk+0AGunTEKUJeLiHRuCRHopw7rCUBJhQJdRDqvhAj0Hl2SAdh/sCrOlYiIxE9CBHqvrsFAv+HF5XGuREQkfhIi0HuEAr24XF0uItJ5tRjoZvYPM9ttZiub2G5m9mczyzWz5WZ2ctuX2bxAxJwuzmmSLhHpnFpzhP4oMLOZ7ecBo0I/VwAPHnlZh+5LJw0EoLJGV4uKSOfUYqA7594F9jXT5ELgcRf0MZBpZv3bqsDWGjcwA4DySgW6iHRObdGHPhDIi1jOD61rxMyuMLMcM8spKChogz9dLy05eLOL0ir1o4tI59ShJ0Wdcw8757Kdc9lZWVlt+txdQoFepqtFRaSTaotA3wYMjlgeFFrXoeruL6rL/0Wks2qLQH8FuDQ02mUiUOic29EGz3tI6o7Qy3XDaBHppJJaamBmTwFTgd5mlg/8CggAOOceAmYDs4BcoBT4bnsV25y6aXQXbtlP9tCe8ShBRCSuWgx059wlLWx3wA/brKLDlJWeAsBvX1/LVVNHxLkaEZGOlxBXikJ9oIuIdFYJE+gpSf7w4zdW7YxjJSIi8ZEwgQ4w64R+APy/fy2KcyUiIh0voQJ90vBe8S5BRCRuEirQk/wJ9XJERA5JQiVgTW39TIvvb9jDtHvns3p7URwrEhHpOAkb6A/Mz2XTnoP8ce76OFYkItJxEirQqyMC/cONewEo0U0vRKSTSKhA75+R2mhdleZHF5FOIqEC/bxx/Xjom9E3TKqq1R2MRKRzSKhANzNmHNc3at3+g5VxqkZEpGMlVKBD46GLe0sq4lSJiEjHSrhAb+hgZQ2V1epHF5HEl/CBDlBaqZEuIpL4EjLQf/eVE6OWN+85qL50EUl4CRnoX80eHLX8xQc+5KQ75rBH/ekiksASMtCbkn3nXIrKq3h5yTZeWJQf73JERNpUi3cs8qp5P5vK2h1FXPXk4qj1uwrLufaZpQB8+ZRB8ShNRKRdJOwR+rDeXcnsktxo/e5idbuISGJK2EAH+MzQHo3WXfqPT8KPF23d15HliIi0q4QO9CS/j7u+OC5qXeSMjF9+8KOOLklEpN0kdKADDMxMi3cJIiIdIuED/czRWdxw3ph4lyEi0u4SPtDNjCvPHBHvMkRE2l3CB3qdkX26xbsEEZF21WkCfe5Pz4y5XhN3iUiiaFWgm9lMM1tnZrlmdkOM7d8xswIzWxr6+X7bl3rklt16TqMbYCzcsk+Td4lIQmgx0M3MD/wVOA8YC1xiZmNjNH3GOTch9PNIG9fZJjK6BMhKT4la941HFvDz55aHl8uraqgNDW2ct3Y3v39jbYfWKCJyuFpzhH4qkOuc2+ScqwSeBi5s37LaT5Kv8Uv+vxU7yN1dQt6+Usbc8jp3z17D6u1FfPfRhfx13sY4VCkicuhaM5fLQCAvYjkfOC1Guy+b2WeB9cBPnHN5MdrEnd9nMdfPuO+d8ONnFubxfu6e8HJldS3JSZ3mdIOIeFRbpdR/gaHOuROBOcBjsRqZ2RVmlmNmOQUFBW30pw9Nwy6XWIZldaWwrCq8vGlPCVv3HmzPskREjlhrAn0bEDnB+KDQujDn3F7nXN2sV48Ap8R6Iufcw865bOdcdlZW1uHUe8T6dk9l/s+msuGu85psszy/kB2F5eHlmfe/x5m/nw8E71H61f/9iNzdJY1+r6i8KuqDQESkI7Um0BcCo8xsmJklAxcDr0Q2MLP+EYufB9a0XYltb2jvrgT8h/7l5G/vbuKxj7byyeZ9zLjvHR6Ynxu1ffyv32T8r99sqzJFRA5Ji6nmnKsGfgS8QTCon3XOrTKz283s86FmPzazVWa2DPgx8J32Kri93fy54xjSs0vMbXfNXsPrK3eEl3/3+jr2lFTgXHBUjHMxf01EpEOYi1MKZWdnu5ycnLj87TpDb/i/8OMnLjuNgT3SGNa7K2WVNRx36+sAPHflJC56qPlZGb8zeSg7C8t5fdVOAF64ajLgOOWYnu1Wu4h0Tma2yDmXHWtbwt6xqDVeuGoyX37wQwC6pPgZ1rsrAGnJfgAuGD+A7GMaz6ne0KMfbolarnvOl34wGb/P8PuMob260jWlU+9uEWlnnTphTokI6y6hEK+z5vaZBPyGmfHJL6fzzCd5jO6XzqAeaQzMTGPC7XNafP7vPrqQA6X1J0nfvu5MhmcF55RxzvHRpr1MGt4Ls/qhlBXVNVz37DJ+evbocNvdReXUOEf/DE0FLCJN69SBHqln1+jb1aVFBHyf9FSunj4qanuPLgH2lzY/ouVAg+0vLdnGtv1lfOf0oXz+Lx8AcMVnh/P9KcNYua2Q11fu5ILxA3h1+Q62HyjjxR+cTu7uYmbc9y4Am+6eha+JcfQiIp26Dx3g+ueX8dGmvbx3/bRD+r2Jd7/FzqJyAn6jqsax5vaZzF2zi6ufWnLINXRLSaKkovF8MicMzGDFtsLwcrLfx8/PPZaZ4/rxXE4e047ry4TBmc0+9wPzcxmYmcaFEwYecl0icvRprg+90wc6BLs/Irs9WmP9rmJmr9jBZWcMw2dG15SkqJOpU0b15r0Ne1p4liO35Z7PsXJbIRlpAfw+Y0BmGv94fzNPLtjKW9dNDZ/43XLP52L+/oZdxQzu2YXUQHSX0/6DlRSWVTE0dF5BRI4OOinagkMNc4DRfdMZ3Tc9al1asp+Th2Sy+NMDDOoRHPp4yamDyd9f1m7hnr+/lPP/5/3w8tdPG8K/F3wKwMUP14/OufnlFdz5hROorK7l+ueXceXUEQzMTOPsP77LhRMG8KeLTwo/322vrCJ/fxlrdxaTc/MMMtMCJDUYt3+wohq/zxp9EHzpgWBX0os/OL3JmpfnH2DDrhK+fMqgI3vxIhJFR+htbMGmvfzkmaX89+ozeOyjrVw4YQDDenXliQVbufU/q4DgMMdHP9zCjOP6MnfNLiB4Ura0sib8PF/NHsSzOfkAXHf2aP4wZ33Mv3ds33TW7SpuVW0nDclkyacHGq33Gdz5hRMor6phzY4inluU36jN2jtm8s76An772loe+96pTPndPAb3TOPYvt0ZntWVm2YdB9DiN4LWthGR2HSE3oFOG96LD2+cDsBPzx4dXn/ppKFMHtGL/P1lTBmVxYmDMvj8+AEsyz9AemqA/Qcr+drDHwPBAP/RtJHMX1dAWrKfq6ePIrNrMre8vDL8fB/dOI1Jv3m71WEOxAxzgFoHN720otnf/d6jC/lw414ApvxuHgB5+8rI21cGa+Dc4/sxpl96zN+trK7lg9w9bCwooai8/lxBdU1t+Mj/3jfWcUyvLlyUHZxl4qUl+QT8Ps4/cUBUeyDq28J/lm5jwuBMjunVMV1D5VU13PPaWq6ZPooeDU6ki8SbAr0DjeyTzsg+wdD70snB7oa6i4+cc9x70XjOGNmbfhmpALz/i2nU9QZ9YcIAbnl5JZldAvz16yfTPyMtfKTfEerCvClffvBDfv3548PLmwpKuPKJRazf1XjOmzqFZVX06pZCba3jL/OC0yiM7pvOmP7p/OSZZQCMH5RJWrKf4vJqzrp3Pr27JfPU5RMprqjmhIEZXPP0UtJTknj1x2fwrb9/ws/PPZYLxtd/CNw3Zz1/fmsDm+6ehRlU1tSSklTfTTRn9S5KK6tjnjTO21cKwOCIK4dfX7mTRz/cwpzVu/jfb53CuIEZze6XgxXVVNc6MtICUevz95dyyd8+5t/fnxj1/EeDf328lU0FJfzqguNbbixHFXW5eMiHG/dw/ICMqHC44vEcPtq0l+Ly6FEyfp/xw6kj+PPbuQ2f5rBdPmUYf3tvc5s9358unkBRWRVb95byyPtNP2+Sz5g4vFfUlMZQ/y0Fgq+3JnRjkvGDMvj9ReNZsGkvt4S6ud67/ixeWJzP/XM3sPy2c0gL+An4feHun49vnB7+IAX45web+fV/V9OrazKLbjmbwtIqUpN9vL5yJ9c8vTTcbv7PptIvI7XRuYQ6Z/5+Hlv3lrLlns+xt6SC37+xjh9NG8mLi7dx35z1XDV1BL+YOabJ176rqJw3V+3kW5OGNrMn29ahdIk553hl2XbOGdsvaqhvW9sZmiwv8t+os1KXS4KYPKJ3o3UPX5pNba1j+E2zAVh08wxSAn66pSThnOOkY3oweUQv8vaVMeO+dxjbvzvfO2MYpZXV4T79WNICfsqqaqLWtfWc8JHBCNEndCNV17pGYQ5w44v13UR1YQ6wLL+Qc/74blTbum4igBNve5OBmWlclF1/UvbB+bl8e/JQpv3hHT4ztEf4te49WElVTS3jb3+T/hmpUbNwAky9dz4DMlJ5/xfTYl4jsHVvabi+B+Zv5OmFeTy9sP5WAeWhfVxVU8sj723mW5OO4fGPttAnPZUpo3pz2WMLWbmtiGnH9WVgZuMLy+au3sW4gRlRQVddU8szOXnk7Stj3MDuUd1WdTbsKuZfH2/ltguOj6r7v8u2N2rbnAWb93HN00v5avYgbr9wXJMfbEdq4m/eAhp/yNTUOmqdO6zJ9trTwYpquiT7D2vAxZFQoCcAn8/4xmlD+OzoLHp1q5/v3cw469g+AIzs041lt55D97Sk8Jts8ohe9Oyawsl3zOHEQRnsKa5ge2E5b113JiNCV6mu3FYYHkXTPyON9Xeex+ibXwPgri+O44LxA3hl6Xbmr9vN3DW7o+pqKqCbcuVnRxxS+/nrDn9O/W0Hyrh/7obw8mMfbeWxj7YCsHDL/qi2P3hyMUCjMK+zvbCc4TfNZsVt55CeGiB3dzEHK2r4zWv1k45u3lPCq8sbh+U/P9hCeVUty/MPsGp7EftLK3n43U2N2v132XbeWLWTU4b04Obzx7LvYCUPzMvlkfc3Ywabf/M5Ln88hzmrdzU6if7S4m3c9vnjGdyzC++uL6Brip9rn1lK3r4yvn/GcIb0qu/yibyO4vR73uZ7ZwzjsjOGhddt2FXMyD7dwu+hguLgrNnP5uTz3oY9fBQ6f9SUwxki3JyLHvqQxZ8eaPbbRE2to6qmttGHTU2t494313HxZwbHPAezcltwGu2zx/Y9pJr2Hazk5Dvm8IuZY7hq6giqa2oZ+cvXuHbGKK6dMbrlJzgCCvQEcdcXT2ixTUaX6H7cuv78F66axOi+6XRJDh7VR550HDcwg82/mcXcNbuZPqYPPp/x07NHc9+c9aQF/HRPDfDNiccweUSvqEA/YWAGd3/xBO68cBxz1+xizY5irp42kuE3zWbi8J70SU+loLiC88f3Z/3OYlZuL2JwzzReu2YKN720InwCNz01qVF3UlNuOG8MD7+7iScuO40vPfgB5VW1rfq9ppw+shcf5O5lzupdrWp/wm1NT5183bPL2FVUEXPbU5/Uf4i9tGRbzDb3vBa8t+2STw+QnORjcM8u4W4q5+Dd9QXhOhuOiHpr7W7SU5M4rn93fvNa9D1yZ/7pXf7nkpOYNqZPo6DddqCMO15dzWVnDGPLnoPc+soq3l1fQJLPuOuL4/jaZ4ZQFjEya0dhOa+t2EFGWoDJI3tTWlnN22t3h78hPJeTx8+fD96/d2SfbvzxqxMYO6A7OwrL6J4WoHtq9Pszln9+sJmXlmzj8e+dSmaXZBZHnOgffuP/ceWZI7i+QRfWL19awdML89j8m1lRr3HNjiIenL+RRVv28+yVkxr9rboDmYYfFqWV1ewtqWx07iNvXylJfmP19iIgeML+qqkjwhcN3j93A+ef2D/8/117UB+6HLKaWsdrK3cwfUzfcL9pdU0tVz+1hFOO6cHZY/vSt3vsfuW8faX06JpMt2YmKnPO8eryHeTtL+WSzwzhpDvq582pu2Br+pg+vLV2N1/LHswzOcEujMj/8RZu2cc76wrCJ1vrxOo2OXFQBsvzC2no9WunMPP+98LLyUk+KquP7EOirYzu263ZE86HatqYPry9dnfMbU1905pxXB8+3Vcas44lt5zN9S8sZ87qXbz4g8kM7tGFyx5b2Gg/N9z3L1w1iYLiSq58YhHTx/Th79/5DBA9MyrATbPGcMLATC75W3BkWN37AeD9X5wVvg5k/8HK8Ptn9e3n4lzwZPxZ987ngvEDeD40RHfdnTOpqnHsK6lkQGYqfp8x7MZgN+aGu87Dbxbumrrwrx+wLO8Af7hoPE8u2MqvLjie9NQkpv3hnagaTxqSyfQxfVixrZA3VtUfFBzpcF1dKSqeduOLy3nqk2Bov/Kj0/nJM0t56YenU15VQ2ZaMg+9s5HJI3qRPbTxdMV1QfDMFRP59yef8sevTmBjQQlnR/Sxz//ZVNbsKKK4opp+3VO59B+fkJEW4L1fnMWJoaPutXfMJDXgDz/f1dNGMqpvOj8OdVFEfpMYntWVTQXBWxb++/LTeH/DHh6YH/tm4987fRj/+CB4pD1+UAbL8gtJT0miOMZUEJ1Fst9HZWiI6i3nj+Wr2YOa/fbT0NdPG8IvZx3HXbPXRH0Q9e2e0uS3pJQkHxURH9aXnDo4/J4DmDS8F3sPVvDod09l8j1vN/r9WFeGR56oj5R713mNLtQ7FAp08bzCsiqcc2R2ObSx302N2Fi3s5hz73+XU4f2jPq67Zxj38FKenRJxgyG3Tibsf27M/uaKQDcP3c9xeXV3HL+WAAWbd3HEx9/yh8uGs+SvAOYwclDevD8onw+O7o3fdJTo+r4+bnHkru7hJeWbOOFqyZFzZn/Qe4ens3JY9qYPlEnjM86Not5EecLuqcmRY3nr3PSkEwmDe/V5IdHa/1g6ojwc3RN9nMw1K1yx4XH89vX18Wcd6i1vjlxCE983PrzJO2tuZBvL18/bQhfmDCQU4cd3v0SFOjSaX20cS+9uiU3mqahtTYVlNA7PaVV/bvNeWlJPnn7yvhxaNbOmlqHv4mZM9fuLIrq6ll669kUl1dT6xz//GAL2UN78KN/L+GW88dyXL90kvw+lubt51sTh5Ia8PFB7l5ufnkFW/aWMvenn6V7WoA+6ank7i5h3trd3DU7eLL2TxdP4JqnlzKkZxfe+fnUcBfDlns+x+9eX8tTn3zKG9d+llPvDo4w2Xj3LPw+49mFeVz/QrAv/KUfTKZ3txR2F1fw32XbSQ34eeidjXzllEFMHN6Lnz23LOq1/fv7p1FZU8t3/rmw0et+8Bsnc1XoBHRTBmSkMqpvOu+sr/+Aizya/v4Zw5odAttQ5IdXRzqSE6QKdBGPWbezmCE9u5C3vzTmh1FJRXWz5yEKiiv4dF9p1Jz/dZ5dmEdqsp9Z4/rx02eXcfmU4ZwwKIPncvLYU1LJVVNHRLW/8l+LeH3VzvC3nKLyKm56cQU/O+fYZidvq611XP3UEr458Rh6dUvmpSXbuHbGKFKS/Nz2yio+3VfKO+sLgudkrngbleMAAAXGSURBVJnCyD7d+Ou83PDoo6cun8jcNbv4eyigzxjZmye+fxq1tY6V2wvZfqCMM0Zl0S0liQm3v8mB0ioW3DSdHz65mJyt+2PWtPy2c/jxU0uYMDiTVduL+O2XT2T7gTKy0lM4LfTBteCm6dz6n5WcdWwfbnix6SuoJwzOZGle7KuvW/LMFRM5bXivw/pdBbqIHLb2HOu9flcx63YWR13d+5+l28hKTwlfd1FYWsW9b67j8inRQywj7SmpoKisKnxTmMc+3MKvXqm/zmLm8f148JsnNztkcmNBCWWVNVFX/94/d33U8NaBmWlsO1AGwJ1fGMfNoek4Nt09i6LyKt5eu5ufPruMlb8+lxX5heGTtv/90RkM7d0lfC5g3Z0zo65YPhQKdBHpdHJ3F7M8v5CUJD/nHN/3sD+Q6qaPuPX8saSnJvHz55dz+ZRh3DTrOIbdOJuMtADLfnVOzN/dWFBCSXk140P3LXhtxQ62F5ZHje0/VLpSVEQ6nci5k47E5VOGUVpRzSWnDiE14KN7WoAZx/UN3p7ypunNHmnXXaBX57wT+h9xPc3REbqIiIc0d4R+dE2AICIih02BLiKSIBToIiIJQoEuIpIgFOgiIglCgS4ikiAU6CIiCUKBLiKSIOJ2YZGZFQBbD/PXewONbzLZeWl/RNP+iKb9US8R9sUxzrmsWBviFuhHwsxymrpSqjPS/oim/RFN+6Neou8LdbmIiCQIBbqISILwaqA/HO8CjjLaH9G0P6Jpf9RL6H3hyT50ERFpzKtH6CIi0oDnAt3MZprZOjPLNbMb4l1PezOzwWY2z8xWm9kqM7smtL6nmc0xsw2h//YIrTcz+3No/yw3s5Pj+wrah5n5zWyJmb0aWh5mZgtCr/sZM0sOrU8JLeeGtg+NZ93twcwyzex5M1trZmvMbFJnfn+Y2U9C/6+sNLOnzCy1s7w/PBXoZuYH/gqcB4wFLjGzsfGtqt1VA9c558YCE4Efhl7zDcBbzrlRwFuhZQjum1GhnyuABzu+5A5xDbAmYvm3wB+dcyOB/cBlofWXAftD6/8Yapdo/gS87pwbA4wnuF865fvDzAYCPwaynXPjAD9wMZ3l/eGc88wPMAl4I2L5RuDGeNfVwfvgP8DZwDqgf2hdf2Bd6PH/ApdEtA+3S5QfYBDBkJoGvAoYwYtFkhq+T4A3gEmhx0mhdhbv19CG+yID2NzwNXXW9wcwEMgDeob+vV8Fzu0s7w9PHaFT/49VJz+0rlMIfR08CVgA9HXO7Qht2gn0DT3uDPvofuB6oDa03As44JyrDi1Hvubw/ghtLwy1TxTDgALgn6EuqEfMrCud9P3hnNsG3At8Cuwg+O+9iE7y/vBaoHdaZtYNeAG41jlXFLnNBQ8vOsVwJTM7H9jtnFsU71qOEknAycCDzrmTgIPUd68Ane790QO4kOAH3QCgKzAzrkV1IK8F+jZgcMTyoNC6hGZmAYJh/qRz7sXQ6l1m1j+0vT+wO7Q+0ffR6cDnzWwL8DTBbpc/AZlmlhRqE/maw/sjtD0D2NuRBbezfCDfObcgtPw8wYDvrO+PGcBm51yBc64KeJHge6ZTvD+8FugLgVGhM9bJBE92vBLnmtqVmRnwd2CNc+6+iE2vAN8OPf42wb71uvWXhkYzTAQKI756e55z7kbn3CDn3FCC//5vO+e+AcwDvhJq1nB/1O2nr4TaJ8zRqnNuJ5BnZseGVk0HVtNJ3x8Eu1ommlmX0P87dfujc7w/4t2JfxgnPWYB64GNwC/jXU8HvN4zCH5dXg4sDf3MItjP9xawAZgL9Ay1N4IjgTYCKwie7Y/762infTMVeDX0eDjwCZALPAekhNanhpZzQ9uHx7vudtgPE4Cc0HvkZaBHZ35/AL8G1gIrgX8BKZ3l/aErRUVEEoTXulxERKQJCnQRkQShQBcRSRAKdBGRBKFAFxFJEAp0EZEEoUAXEUkQCnQRkQTx/wHIitBvD7ivegAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Here's an example training loop using this two-layer model. Can you do better? \n",
        "# learning_rate = 0.01  (Bethany Lush default parameter)\n",
        "learning_rate = 0.1 # Zahmeeth Sakkaff increased the learning_rate for higher accuracy till 94%\n",
        "num_examples = x_train.shape[0]\n",
        "# batch_size = 10000  (Bethany Lush default parameter)\n",
        "batch_size = 1000 # Zahmeeth Sakkaff reduced the batch_size for higher accuracy till 94%\n",
        "num_batches = int(num_examples / batch_size)\n",
        "# num_epochs = 15 (Bethany Lush default parameter)\n",
        "num_epochs = 15 # Zahmeeth Sakkaff increased the epochs for higher accuracy till 94%\n",
        "losses = numpy.zeros(num_batches*num_epochs,)\n",
        "indices = numpy.arange(num_examples)\n",
        "\n",
        "i = 0\n",
        "for epoch in range(0, num_epochs):\n",
        "    # in each epoch, we loop over all of the training examples\n",
        "    for step in range(0, num_batches):\n",
        "        # grabbing the next batch\n",
        "        offset = step * batch_size\n",
        "        batch_range = range(offset, offset+batch_size)\n",
        "        x_train_batch = x_train[batch_range, :]\n",
        "        y_train_batch = y_train_onehot[batch_range,:]\n",
        "        \n",
        "        # feed the next batch in to do one sgd step\n",
        "        loss, model = learn(model, x_train_batch, y_train_batch, learning_rate)\n",
        "        losses[i] = loss\n",
        "        i += 1\n",
        "\n",
        "    acc = accuracy(model, x_train, y_train)\n",
        "    print(\"epoch %d, loss %.5f, accuracy %.2f\" % (epoch, loss, acc))\n",
        "    \n",
        "    # reshuffle the data so that we get a new set of batches\n",
        "    numpy.random.shuffle(indices)\n",
        "    x_train = x_train[indices,:]\n",
        "    y_train = y_train[indices] # keep this shuffled the same way for use in accuracy calculation\n",
        "    y_train_onehot = y_train_onehot[indices,:]\n",
        "plt.plot(losses)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "id": "a4f274c6",
      "metadata": {
        "id": "a4f274c6",
        "outputId": "d3ae9c1f-13db-4590-e2e1-9076c736d9a8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.94395"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "accuracy(model, x_train, y_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a2dd5728",
      "metadata": {
        "id": "a2dd5728"
      },
      "source": [
        "# Homework: improve the accuracy of this model. "
      ]
    },
    {
      "cell_type": "markdown",
      "id": "faaf0515",
      "metadata": {
        "id": "faaf0515"
      },
      "source": [
        "Update this notebook so that the accuracy is improved. How high can you get it? You could change things directly in the notebook, such as increasing the number of epochs, changing the learning weight, changing the width of the hidden layer, etc. If you're more ambitious, you could also try changing the model definition itself by checking out the associated Python files. For example, you could add more layers to the network. The current notebook has a training accuracy of about 43%, but will vary with randomness."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6e484c13",
      "metadata": {
        "id": "6e484c13"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}