Homework 02 - Group 17.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ee337168",
   "metadata": {},
   "source": [
    "# IANNwTF - Homework 02\n",
    "(by Group 17 - Nils Niehaus, Philipp Bauer, Marlon Dammann)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea1e684f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from math import e as exp\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "980c6168",
   "metadata": {},
   "source": [
    "## 1 - Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "003f3a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    '''The sigmoid activation function'''\n",
    "    return 1/(1+exp**(-x))\n",
    "\n",
    "def sigmoidprime(x):\n",
    "    '''The derivative of the sigmoid activation function'''\n",
    "    return (exp**(-x))/(1+exp**(-x))**2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "657cd754",
   "metadata": {},
   "source": [
    "## 2 - Data Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f4eae061",
   "metadata": {},
   "outputs": [],
   "source": [
    "possible_combinations = np.array([[1,1],[1,0],[0,1],[0,0]])\n",
    "and_labels = np.array([1,0,0,0])\n",
    "or_labels = np.array([1,1,1,0])\n",
    "not_and_labels = np.array([0,1,1,1])\n",
    "not_or_labels = np.array([0,0,0,1])\n",
    "xor_labels = np.array([0,1,1,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b371b39",
   "metadata": {},
   "source": [
    "## 3 - Perceptron\n",
    "(For this task we have imported and rewritten the Perceptron class from last semester of Machine Learning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a9933267",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Perceptron:\n",
    "    \"\"\"Single neuron handling its own weights and bias.\"\"\"\n",
    "\n",
    "    def __init__(self, input_units, act_func=sigmoid, learning_rate=1.):\n",
    "        \"\"\"Initialize a new neuron with its weights and bias.\n",
    "\n",
    "        Args:\n",
    "            input_units (int): Dimensionality of the data coming into this perceptron. \n",
    "                In a network of perceptrons this basically represents the \n",
    "                number of neurons in the layer before this neuron's layer. \n",
    "                Used for generating the perceptron's weights vector, which \n",
    "                not only includes one weight per input but also an additional \n",
    "                bias weight.\n",
    "            act_fun (function): Function to apply on activation.\n",
    "            learning_rate (float): Learning rate applied in backpropagation.\n",
    "        \"\"\"\n",
    "        self.act_func = act_func\n",
    "        self.alpha = learning_rate\n",
    "        self.bias = np.random.randn(1)\n",
    "        self.weights = np.random.randn(input_units)\n",
    "        self.output = None\n",
    "\n",
    "    def forward_step(self, x):\n",
    "        \"\"\"Activate this neuron with a specific input.\n",
    "\n",
    "        Calculate the weighted sum of inputs and apply the activation function.\n",
    "\n",
    "        Args:\n",
    "            x (ndarray): Vector of input values.\n",
    "\n",
    "        Returns:\n",
    "            float: A real number representing the perceptron's activation after \n",
    "            calculating the weighted sum of inputs and applying the \n",
    "            perceptron's activation function.\n",
    "        \"\"\"\n",
    "        # Correct inout in case it is a single number\n",
    "        if type(x) != type(np.array):\n",
    "            x = np.array([x])\n",
    "        if len(x.shape) == 2:\n",
    "            x = x[0]\n",
    "        self.x = x\n",
    "        # Return the activation value\n",
    "        self.output = np.append([self.bias], [weight*input_val for weight, input_val in zip(self.weights,x)])\n",
    "        return self.act_func(np.sum(self.output))\n",
    "\n",
    "    def update(self, delta):\n",
    "        \"\"\"Adapt this neuron's weights by a specific delta.\n",
    "\n",
    "        Args:\n",
    "            delta (float): Error term.\n",
    "        \"\"\"\n",
    "        # Adapt self.weights according to the update rule\n",
    "        deltaBias = delta * sigmoidprime(np.sum(self.output))\n",
    "        self.bias -= self.alpha*deltaBias\n",
    "        for i in range(0, len(self.weights)):\n",
    "            deltaWeight = delta * self.x[i]\n",
    "            self.weights[i] -= self.alpha*deltaWeight"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05187147",
   "metadata": {},
   "source": [
    "## 4 - Multi-Layer Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7a46eaf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiLayerPerceptron:\n",
    "    '''The Multi-Layer Perceptron class'''\n",
    "    def __init__(self,size=(2,4,1)):\n",
    "        self.perceptron_layers = []\n",
    "        for i in range(len(size)):\n",
    "            if i == 0:\n",
    "                self.perceptron_layers.append([Perceptron(1) for j in range(size[i])])\n",
    "            else:\n",
    "                self.perceptron_layers.append([Perceptron(size[i-1]) for j in range(size[i])])\n",
    "        self.perceptron_layers = np.array(self.perceptron_layers)\n",
    "        \n",
    "    def forward_step(self,x):\n",
    "        output = [self.perceptron_layers[0][i].forward_step(np.array(x[i])) for i in range(len(x))]\n",
    "        for i in range(1,len(self.perceptron_layers)):\n",
    "            output = [self.perceptron_layers[i][j].forward_step(output) for j in range(len(self.perceptron_layers[i]))]\n",
    "        return output\n",
    "    \n",
    "    def backprop_step(self, target, output):\n",
    "        '''Iterate over output layer'''\n",
    "        delta = [[]]\n",
    "        for i in range(len(self.perceptron_layers[len(self.perceptron_layers)-1])):\n",
    "            delta[0].append((-1)*(target-output)*sigmoidprime(np.sum(self.perceptron_layers[len(self.perceptron_layers)-1][i].output)))\n",
    "            self.perceptron_layers[len(self.perceptron_layers)-1][i].update(delta[0][i])\n",
    "\n",
    "        '''Iterate over hidden layer to input layer'''\n",
    "        for i in range(len(self.perceptron_layers)-1):\n",
    "            delta.append([])\n",
    "            for j in range(len(self.perceptron_layers[len(self.perceptron_layers)-2-i])):\n",
    "                delta[i+1].append(np.sum([delta[i][k]*self.perceptron_layers[len(self.perceptron_layers)-1][k].weights[j] for k in range(len(self.perceptron_layers[len(self.perceptron_layers)-1]))])*sigmoidprime(np.sum(self.perceptron_layers[len(self.perceptron_layers)-2-i][j].output)))\n",
    "                self.perceptron_layers[len(self.perceptron_layers)-2-i][j].update(delta[i+1][j])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37952cc9",
   "metadata": {},
   "source": [
    "## 5 - Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ee9e1784",
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_epoch(model,dataset,labels):\n",
    "    avg_loss = []\n",
    "    accuracy = []\n",
    "    for data,target in list(zip(dataset,labels)):\n",
    "        output = model.forward_step(data)[0]\n",
    "        \n",
    "        avg_loss += [(target - output)**2]\n",
    "        \n",
    "        if output <= 0.5 and target == 0:\n",
    "            accuracy += [1]\n",
    "        elif output >= 0.5 and target == 1:\n",
    "            accuracy += [1]\n",
    "        else:\n",
    "            accuracy += [0]\n",
    "            \n",
    "        model.backprop_step(target, output)\n",
    "    return np.mean(avg_loss), np.mean(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "422d16f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\marlo\\AppData\\Local\\Temp/ipykernel_13376/188105780.py:10: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  self.perceptron_layers = np.array(self.perceptron_layers)\n"
     ]
    }
   ],
   "source": [
    "model = MultiLayerPerceptron()\n",
    "total_avg_loss = []\n",
    "total_accuracy = []\n",
    "for epoch_num in range(1000):\n",
    "    a,b = training_epoch(model,possible_combinations,and_labels)\n",
    "    total_avg_loss += [a]\n",
    "    total_accuracy += [b]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1da9e79e",
   "metadata": {},
   "source": [
    "## 6 - Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2bf3562c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAGoCAYAAADW2lTlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABMN0lEQVR4nO3deXydZZ3//9c7SZM0TbomLd1baNmpLKEomyyCFRBwZAQcFEXB0cF19AeO8x0ZHL9f921kVEBGHRVEBKcKsgiUfWkLyFIodF/ovu9tks/vj/tOehqa5CTp6cnJeT8fj/PIufdPcjjw5rru67oVEZiZmZlZ8SjJdwFmZmZmtn85AJqZmZkVGQdAMzMzsyLjAGhmZmZWZBwAzczMzIqMA6CZmZlZkXEANDPrYSRdJ+nX+a7DzHovB0AzMzOzIuMAaGbWAUll+a7BzGxfcgA0s4Im6VpJcyVtkjRL0vvS9RWS1ks6MmPfOknbJA1Nl8+T9EK635OSJmXsu0DSNZJeBLZIKmvrWun+pZK+K2m1pPmSrpYUzeFR0gBJP5e0TNJSSf8hqTTL3/F8Sa+kdU6TdFjGtmvS822SNFvSmen6yZJmSNooaYWk73XzT21mvYgDoJkVurnAKcAA4N+BX0saHhE7gDuBSzP2/QDwSESslHQMcAvwCWAI8DNgqqSKjP0vBc4FBkZEQ1vXSve9EngPcDRwLHBhqzp/ATQAE4BjgLOBj3f0y0k6GLgV+BxQB9wD/ElSuaRDgKuB4yOiBng3sCA99IfADyOiP3AQcHtH1zKz4uEAaGYFLSJ+HxFvRkRTRPwOeAOYnG7+LXBJxu4fTNcBXAX8LCKeiYjGiPglsAN4e8b+P4qIxRGxLYtrfYAkcC2JiHXAN5pPImkYcA7wuYjYEhErge+3qq0tFwN3R8QDEbEL+A7QFzgRaAQqgMMl9YmIBRExNz1uFzBBUm1EbI6Ip7O4lpkVCQdAMytokj6c0Y27HjgSqE03PwxUSTpB0jiS1rm70m1jgX9uPi49djQwIuP0iztxrRGt9s98PxboAyzLOPZnwNAsfsURwMLmhYhoSs89MiLmkLQMXgeslHSbpOb6PwYcDLwmabqk87K4lpkVCd/YbGYFS9JY4CbgTOCpiGiU9AIggHT5dpKu3BXAnyNiU3r4YuDrEfH1di4R2V4LWAaMyjh2dMb7xSSti7VpV3JnvAkclVGH0nMvTX/H3wK/ldSfJFR+E/hQRLwBXCqpBPg74A5JQyJiSyevb2a9kFsAzayQ9SMJaasAJH2UpFUu029JulH/gd3dv5CEuX9MWwclqZ+kcyXVdPFatwOflTRS0kDgmuYNEbEMuB/4rqT+kkokHSTpnVn8jrcD50o6U1If4J9JwuSTkg6RdEZ63+J2YBvQlNZ3maS6tMVwfXqupiyuZ2ZFwAHQzApWRMwCvgs8RdLCdxTwRKt9ngG2kHSl/iVj/QySgRs/BtYBc4CPdONaN5GEvBeB50kGazSQ3KcH8GGgHJiVXu8OYDgdiIjZwGXAfwKrgfcC742InST3/30jXb+cpEv5y+mhU4BXJG0mGRBySfO9jGZmioiO9zIzs06R9B7gpxExNt+1mJm15hZAM7N9QFJfSeek8wWOBL7K7gEnZmY9ilsAzcz2AUlVwCPAoST34t0NfDYiNua1MDOzvXAANDMzMysy7gI2MzMzKzK9Zh7A2traGDduXL7LMDMzM+sxZs6cuToi6lqv7zUBcNy4ccyYMSPfZZiZmZn1GJIW7m29u4DNzMzMiowDoJmZmVmRcQA0MzMzKzIOgGZmZmZFxgGwkyKC7bsaO97RzMzMrIdyAOyk8/7zcT596/P5LsPMzMysyxwAO2lQVTmrN+/IdxlmZmZmXeYA2Em11Q6AZmZmVtgcADuptrqC1Zt25rsMMzMzsy5zAOyk2poKtu1qZMuOhnyXYmZmZtYlDoCdVFtdAeBuYDMzMytYDoCdVFtdDjgAmpmZWeFyAOyk5hbAVb4P0MzMzAqUA2An1dW4C9jMzMwKmwNgJw3u5y5gMzMzK2wOgJ3Up7SEQVV9HADNzMysYDkAdoHnAjQzM7NCltMAKGmKpNmS5ki6di/bvyBplqQXJT0oaWzGtnslrZf051zW2BW11RVuATQzM7OClbMAKKkUuAF4D3A4cKmkw1vt9jxQHxGTgDuAb2Vs+zbwoVzV1x21NQ6AZmZmVrhy2QI4GZgTEfMiYidwG3BB5g4R8XBEbE0XnwZGZWx7ENiUw/q6LHkesLuAzczMrDDlMgCOBBZnLC9J17XlY8BfOnMBSVdJmiFpxqpVq7pQYtfUVleweUcD23c17rdrmpmZme0rPWIQiKTLgHqSbt+sRcSNEVEfEfV1dXW5KW4v6lomg3Y3sJmZmRWeXAbApcDojOVR6bo9SHoX8BXg/IgoiERVW+O5AM3MzKxw5TIATgcmShovqRy4BJiauYOkY4CfkYS/lTmsZZ9qfhyc7wM0MzOzQpSzABgRDcDVwH3Aq8DtEfGKpOslnZ/u9m2gGvi9pBcktQRESY8BvwfOlLRE0rtzVWtn7Q6AbgE0MzOzwlOWy5NHxD3APa3W/VvG+3e1c+wpOSytW4ZUp13AvgfQzMzMClCPGARSaCrKSulfWeYWQDMzMytIDoBdlEwG7XsAzczMrPA4AHZRbXUFq9wCaGZmZgXIAbCL6vw8YDMzMytQDoBdVFtd7kEgZmZmVpAcALuotrqCjdsb2NHgx8GZmZlZYXEA7KLammQuwDUeCGJmZmYFxgGwizwZtJmZmRUqB8Auqq3284DNzMysMDkAdlFd2gW8YqMDoJmZmRUWB8AuGta/khLBsvXb8l2KmZmZWac4AHZRn9ISDuhfyRIHQDMzMyswDoDdMGJgX950ADQzM7MC4wDYDUkA3J7vMszMzMw6xQGwG0YO6suyDdtoaop8l2JmZmaWNQfAbhgxsC+7GoNVngrGzMzMCogDYDeMHFgJwFLfB2hmZmYFxAGwG0YOrALwQBAzMzMrKA6A3TByUF8AFq7ZmudKzMzMzLLnANgN1RVljBhQyesrNuW7FDMzM7OsOQB208EH1PD6is35LsPMzMwsaw6A3XTwsBrmrtpMo6eCMTMzswLhANhNE4dWs7OhiYVrtuS7FDMzM7OsOAB208HDagDcDWxmZmYFo8MAKOmzkvor8XNJz0k6O5uTS5oiabakOZKu3cv2L0iaJelFSQ9KGpux7XJJb6Svyzv3a+0/E4ZWI8GryzbmuxQzMzOzrGTTAnhFRGwEzgYGAR8CvtHRQZJKgRuA9wCHA5dKOrzVbs8D9RExCbgD+FZ67GDgq8AJwGTgq5IGZfUb7Wf9Kso47ID+zFi4Nt+lmJmZmWUlmwCo9Oc5wP9ExCsZ69ozGZgTEfMiYidwG3BB5g4R8XBENE+i9zQwKn3/buCBiFgbEeuAB4ApWVwzLyaPH8zMhevYvqsx36WYmZmZdSibADhT0v0kAfA+STVAUxbHjQQWZywvSde15WPAXzpzrKSrJM2QNGPVqlVZlJQbpx86lO27mnhizuq81WBmZmaWrWwC4MeAa4Hj09a6PsBH92URki4D6oFvd+a4iLgxIuojor6urm5fltQp7zhwCAOr+vCH55bkrQYzMzOzbGUTAN8BzI6I9WlQ+1dgQxbHLQVGZyyPStftQdK7gK8A50fEjs4c21OUl5VwyfFjuPfl5by23INBzMzMrGfLJgD+BNgq6W3APwNzgV9lcdx0YKKk8ZLKgUuAqZk7SDoG+BlJ+FuZsek+4GxJg9LBH2en63qsT5x6IAOryvmn3zzH3FWeEsbMzMx6rrIs9mmIiJB0AfDjiPi5pI91dFBENEi6miS4lQK3RMQrkq4HZkTEVJIu32rg95IAFkXE+RGxVtLXSEIkwPUR0aOH2Q7qV85PLzuOK381gyk/eJS3HziEg+qqGdKvnD5lJfQpLaGsREQETQEBRAQREKTrApoieaJIy34Z24lIj0uuGUTG+937Nq9ofjbJ7us0L+/er+X45uvuZXvmcWRcEyD52IQEJQK1vFfLdqFkm0BSy7rWx0hC7RxTKlFakrzKSkRpSQmlJVBakvxtS1rW6y3LpRJlpdq9b8vy7n3KS0soTz+r8rJkv/SfSzMzs15Fkflf873tID0C3AtcAZwCrAT+FhFH5b687NXX18eMGTPyXQYrN27npsfm8dgbq1m8ditbdu7bkcFJOKIlmDQHpuR9slF77Ks2j6PluN0ha4/ljPPwlvMk25LQCs3BsCkjpDal6bE57DbF7tAaGe+bWgXbnqS8rITy0hL6lGqPcLhHWCwtoU9ZCeWZ+5SW0Le8lMo+za8S+qbv+6bLlXssZ6wvL6WyrJQ+pQ6gZmbWPZJmRkR96/XZtABeDHyQZD7A5ZLG0MnBGsVkaP9KvnJuMt1hRLCrMWhoakp+NjZRIrVqCaNlXWarWUlLa1hzS1hxBIHM1srWobEpgoamoKkp+dmY8WrY430TTU3Q0NTUsq31MS3vI2hMP59djU3sbGhq+bmzMfZY3r0+Y7mxiV0NwdZtu9iVbmveb/uuRrbtamT7rmwGzb9VaYno26eUqvJSqivK6FdRRr+KzPdlyfvyPde33rd/ZR9qKssoK/WDf8zMLNFhAExD32+A4yWdBzwbEdncA1j0JFFeJsr9xL2sZbZEZjfdZM8XEexoaGLbzka2NzQmP3c1sW1XIzsyQmLyM3k177t1ZyNbdzSyeWcDW3Ykr6Xrt7e837yjgR0N2QXM6ooyBvTtQ/++fRjQN3nf/Opf2YcBVX0ytievwVXlDOjbh5KS3vFZmJlZosMAKOkDJC1+00j+i/yfkr4UEXfkuDazXkFSS3dvLjQ0NrElIyRu3tGQERAb2bR9Fxu27X5tTH8uWL21Zd22diYxLy0Rg6r6MLhfOYP7lTOkX8Xu99Xle11f6sBoZtajZdMF/BWSOQBXAkiqA/5K8ug2M8uzstISBlSVMKCqT5fPsaOhkY3bGt4SEtdu2cnaLTtZs2Una7fsYO2Wnby6fCNrt+xk/dZdez1XiWBIdQXD+lcwtKay5efQ/hUMa/7Zv5Ih/crdLW1mlifZBMCSVlO0rCG76WPMrEBUlJVSV1NKXU1F1sc0NDaxbuuuNCAm4XDN5p2s3ryDlRt3sGLTdpZv2M6LS9azZsvOtwzyaQ6KB/SvZMTASkYM7MvIgX33+FlbXV4097+ame1P2QTAeyXdB9yaLl8M3JO7ksysEJSVllBXU5GGxpp2993V2LQ7GG7czspNO1iZ/ly2YTvzVm3hsTdWs7XVqPnyspI0DFYyYkASCkcPrmLckCrGDKmirrrCAdHMrAuyGQTyJUnvB05KV90YEXfltiwz6036lJYwfEBfhg/o2+Y+EcHGbQ0sWb+VN9dv583123hz/TaWpD8ffWMVKzft2KMlsaq8lLFD+jF2cBVja6sYO7gf44ZUMba2Hwf0r/S9iGZmbcimBZCI+APwhxzXYmZFTFIyErlqAEeMGLDXfXY0NLJ03TYWrt3KwtVbkp9rtvL6yk089NpKdjbuHhFdXlrCmCFVTKirZuKwaiYMreaguuTVtzw3A3LMzApFmwFQ0iZ2PwRij01ARET/nFVlZrYXFWWlHFhXzYF11XDIntsam4JlG7axaM1WFqzZysI1W5i3eguvr9jE/bOWpxOWJ/NsjhzYlwlDq5lQlwTDicOqOXhYDTWVXR9IY2ZWSNoMgBHR/k09ZmY9SGmJGDWoilGDqjhxwp7bdjQ0smD1Vuas3Jy8ViU/n5q7Zo95FEcP7sthB/TnsOHJ6/Dh/Rk1qK/nQTSzXierLmAzs0JWUVbKIQfUcMgBe/5/bWNTsHTdNt5YuYnXlm9i1rKNvLpsIw+8uqLlXsPqijIOPaCmJRQeNXIAhxxQQ3mZJ0Mws8LV4bOAC0VPeRawmRW+bTsbmb1iE6+mgTB5bWLzjgYgGZ182PD+HD1qAJNGDeRtowdwYG21WwrNrMdp61nADoBmZlloagqWrNvGi0vX8+KSDbyweD0vL93QMnVNdUUZR47sz9tGDeTo0QM5buwghvavzHPVZlbs2gqAWXUBSxoLTIyIv0rqC5RFxKZ9XaSZWU9VUiLGpPMPnjdpBJB0Ic9dtZm/LU5C4YtL1nPLE/PZ1Zj8j/WYwVXUjx1E/bjB1I8bxIQ6txKaWc/QYQugpCuBq4DBEXGQpInATyPizP1RYLbcAmhmPcGOhkZmvbmRmQvXMX3BWmYuXMfqzTsBGNC3D8eNHcRxYwcxefxg3jZqoO8lNLOc6nIXsKQXgMnAMxFxTLrupYg4KheFdpUDoJn1RBHBgjVbmbFgLTMWrGPGwrXMXbUFgL59SqkfN4gTD6rlxIOGcOTIAZ682sz2qe50Ae+IiJ3Nj1uSVMbe5wc0M7NWJDG+th/ja/vx9/WjAVi7ZSfPzl/L0/PW8OTc1Xzz3tcAqKks44TxQzjxoCGcOGEIBw+tcZexmeVENgHwEUn/AvSVdBbwKeBPuS3LzKz3GtyvnClHHsCUIw8AYNWmHWkYXMNTc1fz11dXADCkXzmnHlzHOw+u49SD6xjcrzyfZZtZL5JNF3AJ8DHgbJKngNwH3Bw9bPiwu4DNrLdYun4bT81dw+NvrOLRN1azdstOJJg0aiDvPLiO0w6p422jBrq72Mw65GlgzMwKUFNT8NLSDUybvYppr6/khcXriYCBVX04ZWIdZx46lNMPGcqAKj/GzszeqjuDQF7irff8bQBmAP8REWv2WZXd4ABoZsVg3ZadPDZnNdNmr+TR11exevNOykrE2w8cwlmHD+Osw4cxYmDffJdpZj1EdwLgt4BG4LfpqkuAKmA5cHJEvHcf19olDoBmVmyamoLnF6/ngVkruH/Wcualo4uPGjmAsw4fxtlHDOOQYTU0D+Izs+LTnQD4XEQcu7d1PWk6GAdAMyt2c1ZubgmDzy9aD8DYIVWcc9Rwzps0nMOH93cYNCsy3ZkGplTS5Ih4Nj3R8UBpuq2hg4tOAX6Y7n9zRHyj1fZTgR8Ak4BLIuKOjG3fBM5NF78WEb/LolYzs6I1YWg1E4ZW88nTDmLlxu389dWV3PvKcm58dB4/mTaXA2v7ce6k4Zw3aQSHHFCT73LNLI+yaQE8HrgFqCYZBbwR+DjwCnBuRNzexnGlwOvAWcASYDpwaUTMythnHNAf+CIwtTkASjoX+BzwHqACmAacGREb26rTLYBmZnu3dstO7n15OXe/9CZPzV1DU8DEodWcN2kE504azoSh1fku0cxypMstgBExHThK0oB0eUPG5r2Gv9RkYE5EzEsLuA24AGgJgBGxIN3W1OrYw4FHI6IBaJD0IjClg+uZmdleDO5XzgdPGMMHTxjDqk07uPflZfzpxWX84MHX+f5fX+eIEf15/7GjOP/oEdRWV+S7XDPbD7LpAm5ukTsCqGy+fyQiru/gsJHA4ozlJcAJWdb1N+Crkr5LMuDkdDKCY0ZdV5E8p5gxY8ZkeWozs+JVV1PBh94xjg+9YxwrNm7n7heXcdfzS7n+z7P4+j2vctrBdfzdsaM487ChVPYp7fiEZlaQOgyAkn7K7hB2M3AR8Gwui4qI+9Ou5yeBVcBTJCORW+93I3AjJF3AuazJzKy3Gda/kitOHs8VJ4/n9RWbuPO5pfzx+aU8+Npz9K8s47y3jeD9x47k2DGDPHjErJfJpgXwxIiYJOnFiPj3tFXuL1kctxQYnbE8Kl2XlYj4OvB1AEm/Jbmf0MzMcuDgYTVc+55D+dK7D+HJuau587ml3PXcUn77zCLGDani4uPHcNFxo6ircRexWW+QTQDcnv7cKmkEsAYYnsVx04GJksaTBL9LgA9mU1Q6gGRgRKyRNIlklPD92RxrZmZdV1oiTplYxykT6/jahQ3c+/Jybp++mG/e+xrfvX82Zx0+jEsmj+GUCbWU+FF0ZgUrmwD4J0kDgW8Dz5E8FeSmjg6KiAZJV5M8O7gUuCUiXpF0PTAjIqam3bx3AYOA90r694g4AugDPJZ2OWwELksHhJiZ2X5SXVHGRceN4qLjRjFn5WZ+N30Rd8xcwl9eXs6oQX25uH40Hzh+NMP6V+a7VDPrpHangZFUArw9Ip5MlyuAylYjgXsETwNjZpZ7Oxoauf+VFdz67CKenLuG0hJx+iFDueztYzh1Yp1bBc16mO48CeT5iDgmZ5XtIw6AZmb714LVW7ht+mLumLmY1Zt3Mr62Hx9+x1guOm4UNZV98l2emdG9APgdklG4d0ZHO+eRA6CZWX7saGjkLy8t55dPLeD5RevpV17K+48bxYffMZYJQ/3EEbN86k4A3AT0I5mGZRvJ00AiIvrnotCucgA0M8u/F5es5xdPLuDPf1vGzsYmTp5Qy+UnjuOMQ4dS6u5hs/2uywGwUDgAmpn1HGs27+C26Yv59dMLWbZhO6MH9+WKk8bzgfrR9KvI6hkEZrYPdKcFUMA/AOMj4muSRgPDIyKnk0F3lgOgmVnP09DYxP2zVvDfT8xn+oJ19K8s47K3j+UjJ45jqEcPm+VcdwLgT4Am4IyIOEzSIOD+iDg+N6V2jQOgmVnP9tyiddz82DzufXk5pSXigqNHcuUpB3LIAb5P0CxX2gqA2bTDnxARx0p6HiAi1kkq3+cVmplZr3bsmEH81z8cx6I1W7nlifn8bvpi7pi5hFMPruOqUw7kpAlD/Mg5s/2kJIt9dqVP5ggASXUkLYJmZmadNmZIFdedfwRPffkMvvTuQ3h12UYu+/kznPOjx/nT396ksal33Jtu1pNl0wX8D8DFwLHAL4GLgH+NiN/nvrzsuQvYzKww7Who5H9feJMbH53HnJWbGTekin9850G879iRVJSV5rs8s4LWrVHAkg4FziSZAubBiHh135fYPQ6AZmaFrakpuH/Wcm54eC4vLd3AAf0rufLUA7l08miqyj1y2KwrujMI5EfAbc2Pg+upHADNzHqHiOCxN1Zzw8NzeGb+WgZV9eGKk8bz4XeMY0CVnzBi1hndCYCXk3QBHwLcRRIGe1zScgA0M+t9ZixYy39Nm8tDr62kuiKZQuZjJ4+nrqYi36WZFYRuTwQtaTDwfuASYExETNy3JXaPA6CZWe/1ypsb+Mm0udz90jLKS0u4dPIYPnnaQQzzXIJm7WorAGYzCrjZBOBQYCzw2r4qzMzMrCNHjBjAjz94LA9+4Z28920j+J+nF3LKtx7muqmvsHzD9nyXZ1ZwsukC/hbwPmAu8DvgrohYn/vSOsctgGZmxWPhmi3c8PAc/vDcUkpLxCXHj+aTpx3E8AF9812aWY/SnXsAPwH8ISJW56q4fcEB0Mys+Cxeu5UbHp7DHTOXUCJxcRoERwx0EDSD7k8DMwiYCLTcbBERj+7TCrvJAdDMrHgtXruV/5o2l9/PWIwEH6gfzadOn8BIB0Erct1pAfw48FlgFPAC8HbgqYg4Iwd1dpkDoJmZLVm3OwgC/H39aD512kGMGlSV58rM8qM7AfAl4Hjg6Yg4Op0U+v9GxN/lptSucQA0M7NmS9dv4yfT5nD79CUEwUXHjeJTp01g9GAHQSsu3RkFvD0itqcnqYiI10jmBDQzM+uRRg7sy39ceBTTvnQal04ewx9mLuX070zjmjteZNGarfkuzyzvsnm2zhJJA4E/Ag9IWgcszGVRZmZm+8KIgX25/oIj+dRpE/jpI3P57bOLuOO5JbzvmJFcffoExtX2y3eJZnmR9UTQAJLeCQwA7o2InTmrqgvcBWxmZh1ZsXF7EgSfWURDU3DB0SP49BkTGe8gaL1Ut58E0tM5AJqZWbZWbtzOzx6dx2+eWcjOhiYuOHokV58xgYPqqvNdmtk+tS+eBNKVi06RNFvSHEnX7mX7qZKek9Qg6aJW274l6RVJr0r6kSTlslYzMyseQ/tX8n/OO5zH/r8z+NjJ47n35eWc9b1H+OxtzzNn5aZ8l2eWczkLgJJKgRuA9wCHA5dKOrzVbouAjwC/bXXsicBJwCTgSJJRyO/MVa1mZlac6moq+Mq5h/PYNadz5akH8sCsFZz1/Ue5+rfP8foKB0HrvbIZBNJVk4E5ETEPQNJtwAXArOYdImJBuq2p1bFBMul0OSCgD7Aih7WamVkRq62u4MvvOYyrTjmQmx+fz6+eXMDdLy3jnCOH8+kzJ3DoAf3zXaLZPpXLLuCRwOKM5SXpug5FxFPAw8Cy9HVfRLy6zys0MzPLMKS6gmumHMrj15zBp047iEdeX8WUHzzGJ389k1eXbcx3eWb7TE7vAewqSROAw0iePjISOEPSKXvZ7ypJMyTNWLVq1f4u08zMeqlB/cr50rsP5fFrTufTZ0zg8TdW854fPsZVv5rBy0s35Ls8s27LZQBcCozOWB6VrsvG+0iePLI5IjYDfwHe0XqniLgxIuojor6urq7bBZuZmWUaWFXOP599CI9fcwafPXMiT81bw3n/+TgfvuVZnpq7ht4yk4YVn1wGwOnAREnjJZUDlwBTszx2EfBOSWWS+pAMAHEXsJmZ5cWAqj58/qyDefyaM/jSuw9h1psbuPSmp7nwv57k3peX09TkIGiFJafzAEo6B/gBUArcEhFfl3Q9MCMipko6HrgLGARsB5ZHxBHpCOL/Ak4lGRByb0R8ob1reR5AMzPbX7bvauSOmUu48dF5LFq7lQPr+vGPpx7EBceMoKKsNN/lmbXwRNBmZmb7WENjE395eTk/mTaXWcs2Mqx/BR8/+UAuPWEM1RW5nGjDLDsOgGZmZjkSETz2xmp+Mm0uT81bQ01lGZccP5oPv2McowdX5bs8K2IOgGZmZvvBC4vXc/Nj8/jLy8uJCM4+/AA+etI4Jo8fjB9qZfubA6CZmdl+9Ob6bfzP0wu59dlFrN+6iyNG9OejJ43nvW8b7vsEbb9xADQzM8uDbTsb+eMLS/nvJ+bz+orN1FaXc/Hxo7nk+DHuHraccwA0MzPLo4jgiTlr+MWT83notZUEcMrEOj44eTRnHjaMPqU98tkMVuAcAM3MzHqIN9dv4/YZi/nd9MUs27CdupoKPlA/yq2Cts85AJqZmfUwDY1NTJu9ilufXcTDs5NWwbePH8LfHTuS9xw13FPJWLc5AJqZmfVgS9dv4/czFnPX80tZuGYrlX1KePcRB/C+Y0Zy8oRaytxFbF3gAGhmZlYAIoLnFq3nzueW8OcXl7Fh2y7qaio4b9JwzjlqOMeNGURJiaeTsew4AJqZmRWYHQ2NPPzaSu58binTXl/FzoYmhtZU8J4jD+Cco4ZTP24wpQ6D1g4HQDMzswK2afsuHnptJfe8tIxps1exo6GJupoK3n3EMM48dBjvOGgIlX08v6DtyQHQzMysl9iyo4GHZydh8OHXVrFtVyOVfUo48aBazjh0KGccOpQRA/vmu0zrAdoKgB5eZGZmVmD6VZRx3qQRnDdpBNt3NfLM/LU8/NpKHnxtBQ+9thKAQw+o4ZSJtZx4UC3Hjx/sEcW2B7cAmpmZ9RIRwdxVm3notZU89NpKnlu4np2NTZSViLeNHsiJBw3hxINqOWbMQHcXFwl3AZuZmRWZbTsbmblwHU/OXc0Tc9fw0pL1NAVUlJXwtlEDOXbsII4bO4hjxwxkSHVFvsu1HHAANDMzK3Ibt+/i2XlreXLuGmYuWsesNzewqzHJAeOGVHHs2EEcO2YQk0YN4JADaqgocythoXMANDMzsz1s39XIS0s3MHPhOmYuXMdzC9exZstOAMpKxMHDajhyZH+OHDmAI0YM4PDh/elb7lBYSDwIxMzMzPZQ2aeU48cN5vhxg4HkHsLFa7fx8psbeHnpBl5auoG/vrqS22csAaBEMK62H4cMq2HisBoOGVbDwcOqGVfbjz5+UklBcQA0MzMzACQxZkgVY4ZUcc5Rw4EkFC7bsJ2Xlyah8LXlm3ht+SbufWU5zZ2IfUrFgbXVTBxWzcHDajiwrh/ja5NXVbmjRk/kT8XMzMzaJIkRA/syYmBfzj7igJb123c1MmflZt5YuYnZyzfzxopN/G3Jev784rI9jh/WvyINg9UcmIbC8XX9GD2oivIytxrmiwOgmZmZdVpln1KOHDmAI0cO2GP9tp2NLFizhfmrk9e8VVtYsGYL972ynLXp/YUApSVi5MC+jB7clzGDqxg9uIoxGa8Bffsg+TF3ueIAaGZmZvtM3/JSDhven8OG93/LtvVbd7YEw/mrt7BwzVYWrd3KA7NWsHrzzj32rako2x0KhyQBcfSgJCyOHNTXI5S7yQHQzMzM9ouBVeUcM6acY8YMesu2LTsaWLxuK4vSULhk3TYWrd3KnFWbeXj2SnY0NLXsK8Hw/pV7thoO2d2KOKRfuVsPO+AAaGZmZnnXr6KMQw/oz6EHvLXlsKkpWLV5B4vXJuGw+bV47VYefWMVKzbu2PNc5aVthsNRbj0EchwAJU0BfgiUAjdHxDdabT8V+AEwCbgkIu5I158OfD9j10PT7X/MZb1mZmbW85SUiGH9KxnWv5L6dMqaTNt3NbJkXRoM12xl0dptLFqb3Hv46Bur2L4ru9bDsYOrGFwkrYc5C4CSSoEbgLOAJcB0SVMjYlbGbouAjwBfzDw2Ih4Gjk7PMxiYA9yfq1rNzMyscFX2KWXC0BomDK15y7aIPVsPm+877Ezr4dgh/Rg/pB8jB/WltKR3hMNctgBOBuZExDwASbcBFwAtATAiFqTbmvZ2gtRFwF8iYmvuSjUzM7PeSBJDayoZWlPJcWPbbj1sDobN4XBvrYflpSWMHtyX8bXVjK+tYnxtNeNqqziwtpph/SsKquUwlwFwJLA4Y3kJcEIXznMJ8L29bZB0FXAVwJgxY7pwajMzMytmHbYebtrBgjVbmb96M/NXJz8XrE5aD3dmDEzp26eUcbX90mDYj3FD+nFgXfKzJ3Yr9+hBIJKGA0cB9+1te0TcCNwIybOA92NpZmZm1stJYmj/Sob2r2Ty+D1bD5uagmUbtzN/1Rbmr9mS/Fy9mVeXbeK+V1bQ2LQ7lvSvLGN8bT8OrEsmwz74gBrenTGpdj7kMgAuBUZnLI9K13XGB4C7ImLXPqvKzMzMrJtK0omsRw7sy8kTa/fYtquxiSXrtu3Rajh/9RaembeGu55fysSh1b06AE4HJkoaTxL8LgE+2MlzXAp8eV8XZmZmZpYrfUpLWp6F3NrWnQ2saTXpdT7k7CF8EdEAXE3SffsqcHtEvCLpeknnA0g6XtIS4O+Bn0l6pfl4SeNIWhAfyVWNZmZmZvtTVXnyhJN8U0TvuHWuvr4+ZsyYke8yzMzMzHoMSTMjor71+py1AJqZmZlZz+QAaGZmZlZkHADNzMzMiowDoJmZmVmR6TWDQCStAhbuh0vVAqv3w3Ws+/xZFQZ/ToXBn1Nh8OdUGPbn5zQ2Iupar+w1AXB/kTRjb6NprOfxZ1UY/DkVBn9OhcGfU2HoCZ+Tu4DNzMzMiowDoJmZmVmRcQDsvBvzXYBlzZ9VYcjb5yRpmqScd8NI+oykVyX9JtfXanXd6yR9cR+dzt+nwuDPqTDk/XPyPYBmVrQkTQO+GBGdfoyQpLL0kZfZ7Psa8K6IWNLZ63SHpOuAzRHxnf15XTPr+dwCaGY9mqRxaevZTZJekXS/pL7ptpYWPEm1khak7z8i6Y+SHpC0QNLVkr4g6XlJT0sanHGJD0l6QdLLkianx/eTdIukZ9NjLsg471RJDwEP7qXWL6TneVnS59J1PwUOBP4i6fOt9i+V9G1J0yW9KOkT6frTJD0q6W5JsyX9VFJJuu1SSS+l1/hmxrmmSHpO0t8kZdZ2ePp3mifpMxm/393pvi9LurgbH5GZFaCyfBdgZpaFicClEXGlpNuB9wO/7uCYI4FjgEpgDnBNRBwj6fvAh4EfpPtVRcTRkk4FbkmP+wrwUERcIWkg8Kykv6b7HwtMioi1mReTdBzwUeAEQMAzkh6JiH+UNAU4PSJaT/vwMWBDRBwvqQJ4QtL96bbJwOEk01vdC/ydpCeBbwLHAeuA+yVdCDwB3AScGhHzWwXcQ4HTgRpgtqSfAFOANyPi3LT2AR38Lc2sl3EANLNCMD8iXkjfzwTGZXHMwxGxCdgkaQPwp3T9S8CkjP1uBYiIRyX1TwPf2cD5GffPVQJj0vcPtA5/qZOBuyJiC4CkO4FTgOfbqfFsYJKki9LlASRhdyfwbETMS891a3r+XcC0iFiVrv8NcCrQCDwaEfPT3yWzvrsjYgewQ9JKYFj6N/hu2oL454h4rJ0azawXcgA0s0KwI+N9I9A3fd/A7ltZKts5piljuYk9/93X+kboIGnBe39EzM7cIOkEYEunKm+fgE9HxH2trnNaG3V1Reu/XVlEvC7pWOAc4D8kPRgR13fx/GZWgHwPoJkVsgUk3aEAF7WzX3suBpB0Mkl37AbgPuDTkpRuOyaL8zwGXCipSlI/4H3puvbcB3xSUp/0OgenxwJMljQ+vffvYuBx4Fngnen9jqXApcAjwNPAqZLGp+cZ3PpCmSSNALZGxK+Bb5N0a5tZEXELoJkVsu8At0u6Cri7i+fYLul5oA9wRbruayT3CL6YBrD5wHntnSQinpP0C5KQBnBzRLTX/QtwM0l39nNp2FwFXJhumw78GJgAPEzSvdwk6dp0WSTdu/8LkP4N7kzrXQmc1c51jwK+LamJpFv5kx3UaWa9jKeBMTPrYdIu4C9GRLuh08ysq9wFbGZmZlZk3AJoZmZmVmTcAmhmZmZWZBwAzczMzIqMA6CZmZlZkXEANDMzMysyDoBmZmZmRcYB0MzMzKzIOACamZmZFRkHQDMzM7Mi4wBoZmZmVmQcAM3MzMyKjAOgmZmZWZFxADQzMzMrMg6AZtbrSSrLdw37U7H9vmbWeQ6AZpZXkq6VNFfSJkmzJL0vXV8hab2kIzP2rZO0TdLQdPk8SS+k+z0paVLGvgskXSPpRWCLpLK2rpXuXyrpu5JWS5ov6WpJ0RymJA2Q9HNJyyQtlfQfkkrb+J0mS3oqrWuZpB9LKs/YfoSkByStlbRC0r9k1PAvGTXOlDRa0rjMWtJ9p0n6ePr+I5KekPR9SWuA6yQdJOkhSWvS3+k3kgZmHD9a0p2SVqX7/FhSeVrTURn7DZW0VVJdlz9kM+txHADNLN/mAqcAA4B/B34taXhE7ADuBC7N2PcDwCMRsVLSMcAtwCeAIcDPgKmSKjL2vxQ4FxgYEQ1tXSvd90rgPcDRwLHAha3q/AXQAEwAjgHOBj7exu/UCHweqAXeAZwJfApAUg3wV+BeYER6vgfT476Q1nwO0B+4AtjaxjVaOwGYBwwDvg4I+H/pNQ4DRgPXpTWUAn8GFgLjgJHAbRGxE7gNuCzjvJcCD0bEqizrMLNCEBF++eWXXz3mBbwAXJC+fxcwN2PbE8CH0/c/Ab7W6tjZwDvT9wuAKzpxrYeAT2RsexcQQBlJqNoB9M3YfinwcJa/0+eAuzKOe76N/WY319Nq/bjmWjLWTQM+nr7/CLCogxoubL4uSShdlXm+jP1OABYBSpdnAB/I9z8Xfvnl1759+T4RM8srSR8mafkal66qJmk5A3gYqJJ0ArCCpHXurnTbWOBySZ/OOF05SYtXs8WduNaIVvtnvh8L9AGWSWpeV9L6/BnXORj4HlAPVJGEyJnp5tEkLZF70962jrT+XYcBPyRp8axJ612XcZ2FkbSK7iEinpG0FThN0jKSFsqpXazJzHoodwGbWd5IGgvcBFwNDImIgcDLJN2XREQjcDtJq9mlwJ8jYlN6+GLg6xExMONVFRG3Zlwisr0WsAwYlXHs6Iz3i0laAGszrtU/Io5o41f7CfAaMDEi+gP/knGdxcCBbRy3GDhoL+u3pD+rMtYd0GqfaLX8f9N1R6U1XNaqhjHtDBb5Zbr/h4A7ImJ7G/uZWYFyADSzfOpHElJWAUj6KHBkq31+C1wM/EP6vtlNwD9KOkGJfpLOTe+x68q1bgc+K2lkOljimuYNEbEMuB/4rqT+kkrSQRbvbONaNcBGYLOkQ4FPZmz7MzBc0ufSgS41aQsnwM3A1yRNTH+nSZKGRHL/3VLgsnSgyBXsPSi2rmEzsEHSSOBLGdueJQm830j/bpWSTsrY/mvgfSQh8FcdXMfMCpADoJnlTUTMAr4LPEXSxXsUyX1+mfs8Q9ICNgL4S8b6GSQDN35M0rU5h+ReuK5e6yaSkPci8DxwD8mgj8Z0+4dJuphnpde7AxjO3n0R+CCwKT3v7zLq2AScBbwXWA68AZyebv4eSRC9nyRA/hzom267kiTErQGOAJ5s63dN/TvJYJYNwN0kA2qaa2hMrz+B5H6/JSQhu3n7YuA5ksD8WAfXMbMC1HyTr5mZZZD0HuCnETE237Xkg6RbgDcj4l/zXYuZ7XseBGJmBkjqS9ISdz/JqN+vsnvASVGRNA74O5LpbsysF3IXsJlZQiTdputIuoBfBf4trxXlgaSvkQyO+XZEzM93PWaWG+4CNjMzMysybgE0MzMzKzIOgGZmZmZFptcMAqmtrY1x48bluwwzMzOzHmPmzJmrI6Ku9fpeEwDHjRvHjBkz8l2GmZmZWY8haeHe1rsL2MzMzKzIOACamZmZFZmcBkBJUyTNljRH0rV72f59SS+kr9clrc/Y1pixbWou6zQzMzMrJjm7B1BSKXADyTMvlwDTJU1Nn8cJQER8PmP/T7PnrPPbIuLoXNVnZmZmVqxyOQhkMjAnIuYBSLoNuIDkQep7cynJo5d6tMtufoZZyzbmuwwzMzNrx9CaCv74TydR2ac036X0SLkMgCOBxRnLS4AT9rajpLHAeOChjNWVkmYADcA3IuKPeznuKuAqgDFjxuybqjtw0oRaxtf22y/XMjMzs857fcUmnpm/lnVbdzJ8QN98l9Mj9ZRpYC4B7oiIxox1YyNiqaQDgYckvRQRczMPiogbgRsB6uvr98sz7T552kH74zJmZmbWRbc+u4hn5q/FT7ttWy4HgSwFRmcsj0rX7c0lwK2ZKyJiafpzHjCNPe8PNDMzM9sr5buAApDLADgdmChpvKRykpD3ltG8kg4FBgFPZawbJKkifV8LnETb9w6amZmZtVCaAN0A2LacdQFHRIOkq4H7gFLgloh4RdL1wIyIaA6DlwC3RezRUHsY8DNJTSQh9RuZo4fNzMzM2qK0DTDcB9ymnN4DGBH3APe0WvdvrZav28txTwJH5bI2MzMz66WaWwCd/9rkJ4GYmZlZr+J7ADvmAGhmZma9klsA2+YAaGZmZr2K0lEg4WEgbXIANDMzs16luQvYLYBtcwA0MzOzXsXTwHTMAdDMzMx6lZYA6CbANjkAmpmZWa/SMg9gnuvoyRwAzczMrFeR5wHskAOgmZmZ9VJOgG1xADQzM7NepXkaGGubA6CZmZn1Kp4GpmMOgGZmZtareBqYjjkAmpmZWa/SMgrYCbBNDoBmZmbWq+xuAXQCbIsDoJmZmfUqvgewYw6AZmZm1qt4HsCOOQCamZlZr+Qu4LY5AJqZmVkv40EgHXEANDMzs17F80B3zAHQzMzMehUPAulYTgOgpCmSZkuaI+navWz/vqQX0tfrktZnbLtc0hvp6/Jc1mlmZma9R/Oj4HwPYNvKcnViSaXADcBZwBJguqSpETGreZ+I+HzG/p8GjknfDwa+CtSTTOQ9Mz12Xa7qNTMzs97BLYAdy2UL4GRgTkTMi4idwG3ABe3sfylwa/r+3cADEbE2DX0PAFNyWKuZmZn1Er4HsGO5DIAjgcUZy0vSdW8haSwwHnioM8dKukrSDEkzVq1atU+KNjMzs8LmZwF3rKcMArkEuCMiGjtzUETcGBH1EVFfV1eXo9LMzMyskOx+FrAjYFs6DICS7pR0rqTOhsWlwOiM5VHpur25hN3dv5091szMzGw3twB2KJtQ91/AB4E3JH1D0iFZnns6MFHSeEnlJCFvauudJB0KDAKeylh9H3C2pEGSBgFnp+vMzMzM2uVBIB3rMABGxF8j4h+AY4EFwF8lPSnpo5L6tHNcA3A1SXB7Fbg9Il6RdL2k8zN2vQS4LTLaaSNiLfA1khA5Hbg+XWdmZmbWLrWMAnECbEtW08BIGgJcBnwIeB74DXAycDlwWlvHRcQ9wD2t1v1bq+Xr2jj2FuCWbOozMzMza+YWwI51GAAl3QUcAvwP8N6IWJZu+p2kGbkszszMzKyzPAq4Y9m0AP4oIh7e24aIqN/H9ZiZmZl1y+5RwHkupAfLZhDI4ZIGNi+kAzM+lbuSzMzMzLrP08C0LZsAeGVErG9eSJ/McWXOKjIzMzPrBncBdyybAFiq3cNpmp/xW567kszMzMy6zoNAOpbNPYD3kgz4+Fm6/Il0nZmZmVnP09IC6ATYlmwC4DUkoe+T6fIDwM05q8jMzMysG+RHgXSowwAYEU3AT9KXmZmZWY+2+8Y1a0s28wBOBP4fcDhQ2bw+Ig7MYV1mZmZmXeLngHQsm0Eg/03S+tcAnA78Cvh1LosyMzMz66rmsaseBNK2bAJg34h4EFBELEwf3XZubssyMzMz6xp5EEiHshkEskNSCfCGpKuBpUB1bssyMzMz6xpPA9OxbFoAPwtUAZ8BjgMuAy7PZVFmZmZmXeWJoDvWbgtgOunzxRHxRWAz8NH9UpWZmZlZlzXfA+gI2JZ2WwAjohE4eT/VYmZmZtZtbgHsWDb3AD4vaSrwe2BL88qIuDNnVZmZmZl1Ucs0gE6AbcomAFYCa4AzMtYF4ABoZmZmPZZHAbctmyeB+L4/MzMzKxieB7Bj2TwJ5L/ZSyNqRFyRk4rMzMzMusHTwHQsmy7gP2e8rwTeB7yZm3LMzMzMuseDQDqWTRfwHzKXJd0KPJ7NySVNAX4IlAI3R8Q39rLPB4DrSD6nv0XEB9P1jcBL6W6LIuL8bK5pZmZmxU27h4FYG7JpAWxtIjC0o53SOQRvAM4ClgDTJU2NiFkZ+0wEvgycFBHrJGWed1tEHN2F+szMzKyItbQAug+4TdncA7iJPVtRlwPXZHHuycCciJiXnuc24AJgVsY+VwI3RMQ6gIhYmWXdZmZmZu1y/GtbNl3ANV0890hgccbyEuCEVvscDCDpCZJu4usi4t50W6WkGUAD8I2I+GPrC0i6CrgKYMyYMV0s08zMzHqT3S2A+a2jJ+vwWcCS3idpQMbyQEkX7qPrl5F0KZ8GXArcJGlgum1sRNQDHwR+IOmg1gdHxI0RUR8R9XV1dfuoJDMzMytku+8BdAJsS4cBEPhqRGxoXoiI9cBXszhuKTA6Y3lUui7TEmBqROyKiPnA6ySBkIhYmv6cB0wDjsnimmZmZlbk3ALYsWwC4N72yWbwyHRgoqTxksqBS4Cprfb5I0nrH5JqSbqE50kaJKkiY/1J7HnvoJmZmdleeRqYjmUTAGdI+p6kg9LX94CZHR0UEQ3A1cB9wKvA7RHxiqTrJTVP6XIfsEbSLOBh4EsRsQY4LL3u39L138gcPWxmZmbWluYuYLcAti2blrxPA/8H+B1JmH4A+KdsTh4R9wD3tFr3bxnvA/hC+src50ngqGyuYWZmZpZpdwugE2BbshkFvAW4dj/UYmZmZtZtfhRcx7IZBfxAxshc0vvz7stpVWZmZmZd5HsAO5bNPYC16chfANJJmzt8EoiZmZlZPvlJIG3LJgA2SWqZZVnSWByqzczMrMfys4A7ks0gkK8Aj0t6hOQvegrp0zfMzMzMehrPA9ixbAaB3CvpWODt6arPRcTq3JZlZmZm1jVu/+tYNi2AAI3ASqASOFwSEfFo7soyMzMz6xqlTYCeBqZtHQZASR8HPkvyKLcXSFoCnwLOyGllZmZmZl3gaWA6ls0gkM8CxwMLI+J0kmfyrs9lUWZmZmZd5XsAO5ZNANweEdsBJFVExGvAIbkty8zMzKxrWh4Fl+c6erJs7gFckk4E/UfgAUnrgIW5LMrMzMysq3a3ADoCtiWbUcDvS99eJ+lhYABwb06rMjMzM+smx7+2ZTsKGICIeCRXhZiZmZntC2oZBZLXMnq0bO4BNDMzMysYngamYw6AZmZm1qt4GpiOZRUAJY2V9K70fV9JNbkty8zMzKxrWgaB5LeMHq3DACjpSuAO4GfpqlEkI4LNzMzMepyWaWCcANuUTQvgPwEnARsBIuINYGguizIzMzPrLt8D2LZsAuCOiNjZvCCpDLeqmpmZWQ/VMgrY2pRNAHxE0r8AfSWdBfwe+FNuyzIzMzPrGg8C6Vg2AfBaYBXwEvAJ4B7gX3NZlJmZmVmXeRBIhzoMgBHRFBE3RcTfR8RF6fus/qaSpkiaLWmOpGvb2OcDkmZJekXSbzPWXy7pjfR1efa/kpmZmRUztSRAR8C2dPgkEEkv8dYQvQGYAfxHRKxp47hS4AbgLGAJMF3S1IiYlbHPRODLwEkRsU7S0HT9YOCrQH167Znpses6+wuamZlZcfE0MB3L5lFwfwEagebWuUuAKmA58AvgvW0cNxmYExHzACTdBlwAzMrY50rghuZgFxEr0/XvBh6IiLXpsQ8AU4Bbs/qtzMzMrGj5HsCOZRMA3xURx2YsvyTpuYg4VtJl7Rw3ElicsbwEOKHVPgcDSHoCKAWui4h72zh2ZOsLSLoKuApgzJgxWfwqZmZm1tu1PArOCbBN2QwCKZU0uXlB0vEkYQ2goZvXLwMmAqcBlwI3SRqY7cERcWNE1EdEfV1dXTdLMTMzs96gpQUwr1X0bNm0AH4cuEVSNcnfdCPwcUn9gP/XznFLgdEZy6PSdZmWAM9ExC5gvqTXSQLhUpJQmHnstCxqNTMzsyInjwHpUDajgKdHxFHA0cDbImJSRDwbEVsi4vZ2Dp0OTJQ0XlI5yb2DU1vt80fSoCeplqRLeB5wH3C2pEGSBgFnp+vMzMzM2tXyKLg819GTZdMCiKRzgSOAyox+9evbOyYiGiRdTRLcSoFbIuIVSdcDMyJiKruD3iySgSZfah5VLOlrJCES4PrmASFmZmZm7WppAXQEbEs208D8lGTU7+nAzcBFwLPZnDwi7iGZODpz3b9lvA/gC+mr9bG3ALdkcx0zMzOzZn4UXMeyGQRyYkR8GFgXEf8OvIN09K6ZmZlZT+NpYDqWTQDcnv7cKmkEsAsYnruSzMzMzLpObgLsUDb3AP4pnZrl28BzJPdU3pTLoszMzMy6KzwMpE3tBkBJJcCDEbEe+IOkPwOVEbFhfxRnZmZm1lnuAu5Yu13AEdFE8jzf5uUdDn9mZmbWk/lZwB3L5h7AByW9X+5QNzMzswLQMg+gE2CbsgmAnwB+D+yUtFHSJkkbc1yXmZmZWZfsbgF0AmxLh4NAIqJmfxRiZmZmti+5BbBtHbYAKnGZpP+TLo+WNDn3pZmZmZl1nm9a61g2XcD/RTL58wfT5c1kDAwxMzMz60l23wPoJsC2ZDMP4AkRcayk5wEiYp2k8hzXZWZmZtYlLfcAOv+1KZsWwF2SSklHU0uqA5pyWpWZmZlZF7XMA5jXKnq2bALgj4C7gKGSvg48DvzfnFZlZmZm1kXNM9e5BbBt2YwC/o2kmcCZJKH6woh4NeeVmZmZmXXB7hZAJ8C2dBgAJf0IuC0iPPDDzMzMejzfA9ixbLqAZwL/KmmupO9Iqs91UWZmZmZd5YeXdazDABgRv4yIc4DjgdnANyW9kfPKzMzMzLrBDYBty6YFsNkE4FBgLPBabsoxMzMz20fcB9ymbJ4E8q20xe964GWgPiLem/PKzMzMzLpIcgtge7KZCHou8I6IWJ3rYszMzMz2BeEGwPZkcw/gz4BGSZMlndr8yubkkqZImi1pjqRr97L9I5JWSXohfX08Y1tjxvqpnfqtzMzMrKhJ8jQw7chmGpiPA58FRgEvAG8HngLO6OC4UpJnBp8FLAGmS5oaEbNa7fq7iLh6L6fYFhFHd1SfmZmZWWtuAWxfNoNAPksyAnhhRJwOHAOsz+K4ycCciJgXETuB24ALulqomZmZWbZ8D2D7sgmA2yNiO4Ckioh4DTgki+NGAoszlpek61p7v6QXJd0haXTG+kpJMyQ9LenCvV1A0lXpPjNWrVqVRUlmZmZWDITcAtiObALgEkkDgT8CD0j6X2DhPrr+n4BxETEJeAD4Zca2sRFRD3wQ+IGkg1ofHBE3RkR9RNTX1dXto5LMzMys4MmPgmtPNs8Cfl/69jpJDwMDgHuzOPdSILNFb1S6LvPcazIWbwa+lbFtafpznqRpJF3Pc7O4rpmZmRU5gfuA29GZiaCJiEciYmp6T19HpgMTJY2XVA5cAuwxmlfS8IzF84FX0/WDJFWk72uBk4DWg0fMzMzM9sr3ALYvm3kAuyQiGiRdDdwHlAK3RMQrkq4HZkTEVOAzks4HGoC1wEfSww8DfiapiSSkfmMvo4fNzMzM9iq5B9ARsC05C4AAEXEPcE+rdf+W8f7LwJf3ctyTwFG5rM3MzMx6LynfFfRsneoCNjMzMysEngewfQ6AZmZm1uskTwKxtjgAmpmZWa/kFsC2OQCamZlZryM8D2B7HADNzMys95FbANvjAGhmZma9jgcBt88B0MzMzHodyfMAtscB0MzMzHodPwmkfQ6AZmZm1ut4HsD2OQCamZlZr5PMA+gE2BYHQDMzM+t13ALYPgdAMzMz63V8D2D7HADNzMysF5JbANvhAGhmZma9jjwRYLscAM3MzKzXSfKfmwDb4gBoZmZmvY78KLh2OQCamZlZryPfA9guB0AzMzPrlTwPYNscAM3MzKzXcRdw+xwAzczMrNcRHgLSnpwGQElTJM2WNEfStXvZ/hFJqyS9kL4+nrHtcklvpK/Lc1mnmZmZ9S6S7wFsT1muTiypFLgBOAtYAkyXNDUiZrXa9XcRcXWrYwcDXwXqSQL8zPTYdbmq18zMzHoX3wPYtpwFQGAyMCci5gFIug24AGgdAPfm3cADEbE2PfYBYApwa45qNTMzs15EguUbtjNt9sp8l/IWVeVlTB4/OK815DIAjgQWZywvAU7Yy37vl3Qq8Drw+YhY3MaxI1sfKOkq4CqAMWPG7KOyzczMrND1r+zDk3PX8OTcNfku5S0OrOvHQ/98Wl5ryGUAzMafgFsjYoekTwC/BM7I9uCIuBG4EaC+vt7tvGZmZgbArz42mUVrt+a7jL2qKMv/GNxcBsClwOiM5VHpuhYRkRnLbwa+lXHsaa2OnbbPKzQzM7Neqba6gtrqinyX0WPlMoJOByZKGi+pHLgEmJq5g6ThGYvnA6+m7+8DzpY0SNIg4Ox0nZmZmZl1U85aACOiQdLVJMGtFLglIl6RdD0wIyKmAp+RdD7QAKwFPpIeu1bS10hCJMD1zQNCzMzMzKx7FL1kkpz6+vqYMWNGvsswMzMz6zEkzYyI+tbr838XopmZmZntVw6AZmZmZkWm13QBS1oFLNwPl6oFVu+H61j3+bMqDP6cCoM/p8Lgz6kw7M/PaWxE1LVe2WsC4P4iacbe+tKt5/FnVRj8ORUGf06FwZ9TYegJn5O7gM3MzMyKjAOgmZmZWZFxAOy8G/NdgGXNn1Vh8OdUGPw5FQZ/ToUh75+T7wE0MzMzKzJuATQzMzMrMg6AZmZmZkXGAbATJE2RNFvSHEnX5rueYiZptKSHJc2S9Iqkz6brB0t6QNIb6c9B6XpJ+lH62b0o6dj8/gbFRVKppOcl/TldHi/pmfTz+J2k8nR9Rbo8J90+Lq+FFxFJAyXdIek1Sa9Keoe/Tz2PpM+n/857WdKtkir9feoZJN0iaaWklzPWdfo7JOnydP83JF2eq3odALMkqRS4AXgPcDhwqaTD81tVUWsA/jkiDgfeDvxT+nlcCzwYEROBB9NlSD63ienrKuAn+7/kovZZ4NWM5W8C34+ICcA64GPp+o8B69L130/3s/3jh8C9EXEo8DaSz8vfpx5E0kjgM0B9RBwJlAKX4O9TT/ELYEqrdZ36DkkaDHwVOAGYDHy1OTTuaw6A2ZsMzImIeRGxE7gNuCDPNRWtiFgWEc+l7zeR/MdqJMln8st0t18CF6bvLwB+FYmngYGShu/fqouTpFHAucDN6bKAM4A70l1af07Nn98dwJnp/pZDkgYApwI/B4iInRGxHn+feqIyoK+kMqAKWIa/Tz1CRDwKrG21urPfoXcDD0TE2ohYBzzAW0PlPuEAmL2RwOKM5SXpOsuztFvjGOAZYFhELEs3LQeGpe/9+eXPD4D/D2hKl4cA6yOiIV3O/CxaPqd0+4Z0f8ut8cAq4L/TrvqbJfXD36ceJSKWAt8BFpEEvw3ATPx96sk6+x3ab98tB0AraJKqgT8An4uIjZnbIpnjyPMc5ZGk84CVETEz37VYu8qAY4GfRMQxwBZ2d1UB/j71BGlX4AUkgX0E0I8ctQ7ZvtfTvkMOgNlbCozOWB6VrrM8kdSHJPz9JiLuTFevaO6KSn+uTNf788uPk4DzJS0guW3iDJJ7zQamXViw52fR8jml2wcAa/ZnwUVqCbAkIp5Jl+8gCYT+PvUs7wLmR8SqiNgF3EnyHfP3qefq7Hdov323HACzNx2YmI62Kie58XZqnmsqWul9LD8HXo2I72Vsmgo0j5q6HPjfjPUfTkdevR3YkNEsbzkSEV+OiFERMY7kO/NQRPwD8DBwUbpb68+p+fO7KN2/x/wfc28VEcuBxZIOSVedCczC36eeZhHwdklV6b8Dmz8nf596rs5+h+4DzpY0KG3xPTtdt8/5SSCdIOkckvuZSoFbIuLr+a2oeEk6GXgMeInd95b9C8l9gLcDY4CFwAciYm36L8sfk3SXbAU+GhEz9nvhRUzSacAXI+I8SQeStAgOBp4HLouIHZIqgf8huadzLXBJRMzLU8lFRdLRJAN1yoF5wEdJGgn8fepBJP07cDHJTAjPAx8nuUfM36c8k3QrcBpQC6wgGc37Rzr5HZJ0Bcl/zwC+HhH/nZN6HQDNzMzMiou7gM3MzMyKjAOgmZmZWZFxADQzMzMrMg6AZmZmZkXGAdDMzMysyDgAmlnRkjRNUv1+uM5nJL0q6Te5vlar614n6Yv785pmVhjKOt7FzMxak1SW8fzVjnwKeFdELMllTWZm2XILoJn1aJLGpa1nN0l6RdL9kvqm21pa8CTVpo+cQ9JHJP1R0gOSFki6WtIXJD0v6WlJgzMu8SFJL0h6WdLk9Ph+km6R9Gx6zAUZ550q6SHgwb3U+oX0PC9L+ly67qfAgcBfJH2+1f6lkr4tabqkFyV9Il1/mqRHJd0tabakn0oqSbddKuml9BrfzDjXFEnPSfqbpMzaDk//TvMkfSbj97s73fdlSRd34yMyswLkFkAzKwQTgUsj4kpJtwPvB37dwTFHkjwBoRKYA1wTEcdI+j7wYZKn+gBURcTRkk4FbkmP+wrJY7OukDQQeFbSX9P9jwUmRcTazItJOo7k6RknAAKekfRIRPyjpCnA6RGxulWNHyN5BNTxkiqAJyTdn26bDBxO8vSAe4G/k/Qk8E3gOGAdcL+kC4EngJuAUyNifquAeyhwOlADzJb0E5KnD7wZEeemtQ/o4G9pZr2MA6CZFYL5EfFC+n4mMC6LYx6OiE3AJkkbgD+l618CJmXsdytARDwqqX8a+M4Gzs+4f66S5FFOAA+0Dn+pk4G7ImILgKQ7gVNIHs3VlrOBSZKan+M6gCTs7gSebX5sV/qIqZOBXcC0iFiVrv8NcCrQCDwaEfPT3yWzvrsjYgewQ9JKYFj6N/hu2oL454h4rJ0azawXcgA0s0KwI+N9I9A3fd/A7ltZKts5piljuYk9/93X+nmYQdKC9/6ImJ25QdIJwJZOVd4+AZ+OiD0e9p4+N3lvdXVF679dWUS8LulY4BzgPyQ9GBHXd/H8ZlaAfA+gmRWyBSTdoQAXtbNfey4GkHQySXfsBuA+4NPpA9uRdEwW53kMuFBSlaR+wPvSde25D/ikpD7pdQ5OjwWYLGl8eu/fxcDjwLPAO9P7HUuBS4FHgKeBUyWNT88zuPWFMkkaAWyNiF8D3ybp1jazIuIWQDMrZN8Bbpd0FXB3F8+xXdLzQB/ginTd10juEXwxDWDzgfPaO0lEPCfpFyQhDeDmiGiv+xfgZpLu7OfSsLkKuDDdNh34MTABeJike7lJ0rXpski6d/8XIP0b3JnWuxI4q53rHgV8W1ITSbfyJzuo08x6GUV0tVfBzMxyIe0C/mJEtBs6zcy6yl3AZmZmZkXGLYBmZmZmRcYtgGZmZmZFxgHQzMzMrMg4AJqZmZkVGQdAMzMzsyLjAGhmZmZWZP5/Nn4Qx15rY9EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(nrows=2, ncols=1, sharex=True, figsize=(9, 6))\n",
    "axes[0].set(title=\"average loss\")\n",
    "axes[1].set(title=\"average accuracy\")\n",
    "\n",
    "x = np.array(list(range(1,1001)))\n",
    "y_1 = np.array(total_avg_loss)\n",
    "y_2 = total_accuracy\n",
    "\n",
    "axes[0].set(\n",
    "        title=\"average loss\",\n",
    "        ylabel=\"average loss\", \n",
    "        xlabel=\"number of epochs\"\n",
    ")\n",
    "\n",
    "axes[1].set(\n",
    "        title=\"average accuracy\",\n",
    "        ylabel=\"average accuracy\", \n",
    "        xlabel=\"number of epochs\"\n",
    ")\n",
    "\n",
    "axes[0].plot(x, y_1)\n",
    "axes[1].plot(x, y_2)\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74e1670e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}