Homework_10_Group_17_finished.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e42221f2",
   "metadata": {},
   "source": [
    "# IANNWTF - Homework 10\n",
    "(by Group 17 - Nils Niehaus, Marlon Dammann, Philipp Bauer)\n",
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "33cdb34f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow_text import RegexSplitter\n",
    "\n",
    "from collections import Counter\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.spatial.distance import cosine as CosineDistance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2a634e6",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d0ed24d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The First Book of Moses:  Called Genesis\\n\\n\\n1:1 In the beginning God created the heaven and the earth'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try:\n",
    "    import google.colab\n",
    "    from google.colab import drive\n",
    "    os.chdir('drive/MyDrive')\n",
    "    with open('./bible.txt','r') as file:\n",
    "        raw_text = file.read()\n",
    "except:\n",
    "    with open('./bible.txt','r') as file:\n",
    "        raw_text = file.read()\n",
    "raw_text[0:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e99e8c2",
   "metadata": {},
   "source": [
    "In the following we will use only regex modifications and string class methods since the bible is, compared to other data, relatively easy to preprocess."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8819a4b",
   "metadata": {},
   "source": [
    "## Word Embeddings\n",
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "add89f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_text = raw_text.lower()\n",
    "preprocessed_text = re.sub(r'(\\n)|([0123456789:;,\\.?!\\)\\(\\\"\\']+)',\"\",preprocessed_text) # Replaces all unusual characters with a space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c18c35f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=string, numpy=\n",
       "array([b'the', b'first', b'book', b'of', b'moses', b'called', b'genesis',\n",
       "       b'in', b'the', b'beginning'], dtype=object)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter = RegexSplitter(r' ')\n",
    "word_split = text_splitter.split(preprocessed_text)\n",
    "word_split[0][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3437a228",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_n_frequent(n,word_split):\n",
    "    c = Counter(word_split)\n",
    "    return [word for word,count in c.most_common(n)], [count for word,count in c.most_common(n)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4710b30d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected the 10000 most common words.\n",
      "[b'the', b'and', b'of', b'to', b'in', b'that', b'he', b'shall', b'i', b'for']\n",
      "[0.07510630543068886, 0.06263178498629841, 0.04165822972772311, 0.01626236855249126, 0.015378177351206146, 0.015239136597416271, 0.012775550425896677, 0.011533633013404609, 0.011112461021342081, 0.010947771973163784]\n"
     ]
    }
   ],
   "source": [
    "subset = 10000\n",
    "tokens, frequencies = get_n_frequent(subset,word_split[0].numpy())\n",
    "frequencies = [freq/int(word_split[0].shape[0]) for freq in frequencies]\n",
    "print('Selected the',len(tokens),'most common words.')\n",
    "print(tokens[:10])\n",
    "print(frequencies[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c36ae5c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2idx(words):\n",
    "    return {words[i] : i for i in range(len(words))}\n",
    "word2idx = word2idx(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9cd2f40b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_context_pairs(tokens,word_split,window=4):\n",
    "    context_pairs = {token : [] for token in tokens}\n",
    "    for token in tqdm(tokens,desc='Retrieving context words for tokens:'):\n",
    "        context_words = []\n",
    "        for i in range(len(word_split)):\n",
    "            if word_split[i] == token:\n",
    "                for s in range(window):\n",
    "                    try:\n",
    "                        if word_split[i+s] in tokens:\n",
    "                            context_words += [word_split[i+s]]\n",
    "                    except IndexError:\n",
    "                        pass\n",
    "                    except Exception as e:\n",
    "                        print(e)\n",
    "        context_pairs[token] = list(set(context_words))\n",
    "    return context_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2ccc2e40",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrieving context words for tokens:: 100%|██████████████████████████████████████| 10000/10000 [09:41<00:00, 17.19it/s]\n"
     ]
    }
   ],
   "source": [
    "context_pairs = get_word_context_pairs(tokens,word_split[0].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5189aa2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_pairs = [[(token,target) for target in context_pairs[token]] for token in list(context_pairs.keys())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "321682c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_pairs = [pair for sublst in target_pairs for pair in sublst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "48e08e98",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_pairs = [(word2idx[a],word2idx[b]) for a,b in target_pairs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1c8ccc7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_raw = tf.data.Dataset.from_tensor_slices(target_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6e3cd3e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_dataset(data):\n",
    "    data = data.map(lambda datum: (datum[0],datum[1]))\n",
    "    '''We shuffle, take a batch size of 64 and prefetch 20 elements.'''\n",
    "    data = data.shuffle(10000000)\n",
    "    data = data.batch(64)\n",
    "    data = data.map(lambda a, b: (a,tf.reshape(b,(-1,1))))\n",
    "    data = data.prefetch(20)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "a545e09d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = data_raw.apply(preprocess_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c14b947",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "7a88918f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SkipGram(tf.keras.layers.Layer):\n",
    "    def __init__(self, vocab_size=10000, embedding_size=64, activation=tf.nn.softmax,optimizer=tf.keras.optimizers.Adam(0.001)):\n",
    "        super(SkipGram, self).__init__()\n",
    "        \n",
    "        self.optimizer = optimizer\n",
    "        self.vocab = vocab_size\n",
    "        self.embedding_size = embedding_size\n",
    "        self.activation = activation\n",
    "        self.build()\n",
    "\n",
    "    def build(self,input_shapes=None):\n",
    "        self.w = tf.Variable(tf.random.normal([self.vocab,self.embedding_size]))\n",
    "        self.b = tf.Variable(tf.zeros([self.vocab]))\n",
    "        self.embedding = tf.Variable(tf.random.uniform([self.vocab,self.embedding_size]))\n",
    "\n",
    "    def call(self, word, target):\n",
    "        embedding = tf.nn.embedding_lookup(self.embedding, word)\n",
    "        nce_loss = tf.nn.nce_loss(weights=self.w, biases=self.b, inputs=embedding, labels=target, num_sampled=64, num_classes=self.vocab)\n",
    "        \n",
    "        return tf.reduce_mean(nce_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63000420",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "5f3932de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'holy' is part of subset: True\n",
      "b'god' is part of subset: True\n",
      "b'wine' is part of subset: True\n",
      "b'angel' is part of subset: True\n",
      "b'devil' is part of subset: True\n",
      "b'sin' is part of subset: True\n",
      "b'commandments' is part of subset: True\n"
     ]
    }
   ],
   "source": [
    "'''Trace'''\n",
    "epoch_loss = {}\n",
    "test_words = [b'holy',b'god',b'wine',b'angel',b'devil',b'sin',b'commandments']\n",
    "for word in test_words:\n",
    "    print(f'{word} is part of subset: {word in tokens}')\n",
    "\n",
    "'''Hyperparameters'''\n",
    "epochs = 10\n",
    "learning_rate = 0.001\n",
    "\n",
    "'''Model'''\n",
    "model = SkipGram(optimizer=tf.keras.optimizers.Adam(learning_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "3bf777c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nearest_words_print(model):\n",
    "    global test_words\n",
    "    global word2idx\n",
    "    embedding = model.embedding.numpy()\n",
    "    results = []\n",
    "    for test_word in test_words:\n",
    "        idx = word2idx[test_word]\n",
    "        distances = []\n",
    "        for i in range(len(embedding)):\n",
    "            if idx != i:\n",
    "                distances += [CosineDistance(embedding[i], embedding[idx])]\n",
    "        closest = np.argmin(distances)\n",
    "        results += [{idx : word for word, idx in word2idx.items()}[closest]]\n",
    "    print('############################################################')\n",
    "    for i in range(len(test_words)):\n",
    "        print(f'Closest target for word: \\'{test_words[i].decode(\"utf-8\")}\\' was \\'{results[i].decode(\"utf-8\")}\\'.')\n",
    "    print('############################################################')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "c9f13baa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(dataset,model,epochs):\n",
    "    global epoch_loss\n",
    "    for i in range(epochs):\n",
    "        epoch_loss[i] = []\n",
    "        for datum in tqdm(dataset,desc='Current epoch: ' + str(i)):\n",
    "            train_step(datum,model,i)\n",
    "        epoch_loss[i] = np.mean(epoch_loss[i])\n",
    "        nearest_words_print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "f6bd2290",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_step(data,model,i):\n",
    "    global epoch_loss\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = model(data[0],data[1])\n",
    "        epoch_loss[i] += [loss.numpy()]\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        model.optimizer.apply_gradients(zip(gradients, model.trainable_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "52a57b71",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 0: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:03<00:00,  3.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was yonder.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 1: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was yonder.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 2: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was yonder.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 3: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 4: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 5: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 6: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 7: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 8: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Current epoch: 9: 100%|████████████████████████████████████████████████████████████████| 10/10 [00:02<00:00,  3.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############################################################\n",
      "Closest target for word: holy was chased.\n",
      "Closest target for word: god was wert.\n",
      "Closest target for word: wine was wisdomand.\n",
      "Closest target for word: angel was himself.\n",
      "Closest target for word: devil was rooms.\n",
      "Closest target for word: sin was freewoman.\n",
      "Closest target for word: commandments was utmost.\n",
      "############################################################\n"
     ]
    }
   ],
   "source": [
    "training_loop(dataset,model,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "9b4126ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqLklEQVR4nO3deXhV1bnH8e+bgYQxJIwBEhIQgYCAEhAFAaeKlKtWb9XWOl9nr1q9t1bb29pbWztStdYBqxW9Fm2dSp0AtYJWFMIgo4BgwgyBMIQhISTv/eNs0oCAAXKyT3J+n+c5T85Z+wwv59H8stdaey1zd0RERAASwi5ARERih0JBRESqKRRERKSaQkFERKopFEREpJpCQUREqikURBo4Mys0s7PCrkMaB4WCNFr6ZSly5BQKIiEys6SwaxCpSaEgccfMUszsQTNbG9weNLOU4FhbM3vdzLaaWYmZfWBmCcGxu81sjZmVmtkSMzvzEO/fxsz+bmbbzWymmd1vZh/WOO5mdouZLQOWBW0Pmdmq4DWzzOy0Gs+/z8xeMrMXg8+ebWb9D/jYAWY2z8y2Bc9LrevvTeKDQkHi0Q+AIcAAoD8wGPhhcOwuYDXQDugA3Au4mfUEbgUGuXtL4Byg8BDv/wdgJ9ARuDK4HegC4GQgL3g8M6gnA/gz8NcDfrGfD/y1xvHXzCy5xvGLgVFALtAPuOow/36RQ1IoSDy6DPhfd9/o7sXAT4DLg2MVQCbQ1d0r3P0DjywQVgmkAHlmluzuhe6+/MA3NrNE4CLgx+6+y90XAeMPUsMD7l7i7rsB3P3/3H2zu+91998Gn9WzxvNnuftL7l4BjAVSiQTbPg+7+1p3LwH+TiRgRI6YQkHiUSegqMbjoqAN4NfA58BkM1thZt8HcPfPgTuA+4CNZvaCmXXiy9oBScCqGm2rDvK8/drM7L/MbHHQ/bMVSAPaHuz57l5F5Gym5uevr3F/F9DiIJ8p8pUUChKP1gJdazzODtpw91J3v8vduwHnAXfuGztw9z+7+7DgtQ788iDvXQzsBbrUaMs6yPOqlycOxg++R6QLKN3dWwPbADvYewRjHF321SxSlxQK0tglm1lqjVsSMAH4oZm1M7O2wI+A/wMwszFmdpyZGZFfzJVAlZn1NLMzggHpMmA3UHXgh7l7JfAKcJ+ZNTOzXsAVX1FjSyJBUgwkmdmPgFYHPGegmV0Y1H8HUA58fBTfh8hhKRSksXuTyC/wfbf7gPuBAmAeMB+YHbQB9ADeAXYA04FH3f0fRPr4fwFsItJV0x645xCfeSuR7p/1wHNEQqj8MDVOAt4GlhLpyirjy11OfwMuAbYQGf+4MBhfEKlTpk12RKLLzH4JdHT3g81Cqs3r7wOOc/fv1GlhIgehMwWROmZmvcysn0UMBq4FXg27LpHa0NWUInWvJZEuo07ABuC3RLp/RGJe1LqPzCwLeJbIBUAOjHP3h4JT4euIDKoB3OvubwavuYfIX1WVwG3uPikqxYmIyEFFMxQygUx3n21mLYFZRK7ivBjY4e6/OeD5eUT+uhpM5C+sd4Djg9kcIiJSD6LWfeTu64B1wf1SM1sMdD7MS84HXnD3cuALM/ucSEBMP9QL2rZt6zk5OXVXtIhIHJg1a9Ymd293sGP1MqZgZjnAicAnwFDgVjO7gsi0wLvcfQuRwKg573o1BwkRM7seuB4gOzubgoKC6BYvItLImFnRoY5FffaRmbUAXgbucPftwGNAdyJrs6wjMghXa+4+zt3z3T2/XbuDBp2IiBylqIZCsIrjy8Dz7v4KgLtvcPfKYP2WJ4l0EQGsYf/lALoEbSIiUk+iFgrBMgFPAYvdfWyN9swaT/sGsCC4PxG4NFjrPpfIlaUzolWfiIh8WTTHFIYSuRx/vpnNDdruBb5lZgOITFMtBG4AcPeFZvYXYBGRdWBu0cwjEZH6Fc3ZRx+y/yqP+7x5mNf8DPhZtGoSEZHD0zIXIiJSTaEgIiLV4jIUNu0o5yd/X8i2XVp5WESkprgMhQ3by3jmo0J+/96ysEsREYkpcRkKfTql8c2BXRg/vZDCTTvDLkdEJGbEZSgA/NfXepKcmMADby0OuxQRkZgRt6HQvlUqN43ozqSFG5i+fHPY5YiIxIS4DQWA64Z3o1NaKve/sYiqKm1LKiIS16GQmpzI3ef2YuHa7bw8e3XY5YiIhC6uQwHg3/p1on9Wa349aQk7y/eGXY6ISKjiPhQSEowfjenNxtJynpi6POxyRERCFfehADCwawZj+mUy7oMVrN26O+xyRERCo1AI3D2qF1UOv560JOxSRERCo1AIZGU049phubw6Zw1zV20NuxwRkVAoFGq4eWR32rZowv2vL8JdU1RFJP4oFGpomZrMnWf3pKBoC2/OXx92OSIi9U6hcIBLBmXRq2NLHnhrMWUV2vhNROKLQuEAiQnGD7+ex+otu3nmo8KwyxERqVcKhYMY1qMtZ/RqzyPvfc6mHeVhlyMiUm8UCodw7+jelFVUMnbK0rBLERGpN1ELBTPLMrN/mNkiM1toZrcH7b82s8/MbJ6ZvWpmrYP2HDPbbWZzg9vj0aqtNo5r34LvDOnKCzNWsmR9aZiliIjUm2ieKewF7nL3PGAIcIuZ5QFTgL7u3g9YCtxT4zXL3X1AcLsxirXVyu1n9qBFShL3v6EpqiISH6IWCu6+zt1nB/dLgcVAZ3ef7O77Vp77GOgSrRqOVXrzJtx2Zg8+WLaJ95cUh12OiEjU1cuYgpnlACcCnxxw6BrgrRqPc81sjplNNbPTDvFe15tZgZkVFBdH/xf1FafkkNu2Ofe/sYiKyqqof56ISJiiHgpm1gJ4GbjD3bfXaP8BkS6m54OmdUC2u58I3An82cxaHfh+7j7O3fPdPb9du3bRLp8mSQncc24vlhfvZMKMlVH/PBGRMEU1FMwsmUggPO/ur9RovwoYA1zmQWe9u5e7++bg/ixgOXB8NOurrbPzOjCkWwa/m7KUbbsqwi5HRCRqojn7yICngMXuPrZG+yjge8B57r6rRns7M0sM7ncDegArolXfkTAz/mdMHlt3V/D795aFXY6ISNRE80xhKHA5cEaNaaajgUeAlsCUA6aeDgfmmdlc4CXgRncviWJ9R6RPpzS+ObAL46cXUrhpZ9jliIhEhTXkqZb5+fleUFBQb5+3cXsZI3/zPqf1aMsTl+fX2+eKiNQlM5vl7gf9JaYrmo9A+1ap3DSiO5MWbmD68s1hlyMiUucUCkfouuHd6JSWyv1vLKKqquGeZYmIHIxC4QilJifyvVG9WLh2Oy/PXh12OSIidUqhcBTO69+J/lmt+fWkJews3/vVLxARaSAUCkchIcH40ZjebCwt54lpMTFrVkSkTigUjtLArhmM6ZfJuGnLWbdtd9jliIjUCYXCMbh7VC+qHH719pKwSxERqRMKhWOQldGMa4fl8uqcNcxdtTXsckREjplC4RjdPLI7bVs04f7XteeCiDR8CoVj1DI1mTvP7klB0RbenL8+7HJERI6JQqEOXDIoi14dW/LAW4spq6gMuxwRkaOmUKgDiQnGD7+ex+otu3nmo8KwyxEROWoKhToyrEdbzujVnkfe+5xNO8rDLkdE5KgoFOrQvaN7U1ZRydgpS8MuRUTkqCgU6tBx7VvwnSFdeWHGSpasLw27HBGRI6ZQqGO3n9mDFilJ3P+GpqiKSMOjUKhj6c2bcNuZPfhg2SbeX1IcdjkiIkdEoRAFV5ySQ06bZtz/xiIqKqvCLkdEpNYUClHQJCmBe0b3ZnnxTibMWBl2OSIitaZQiJKv5XVgSLcMfjdlKdt2VYRdjohIrUQtFMwsy8z+YWaLzGyhmd0etGeY2RQzWxb8TA/azcweNrPPzWyemZ0Urdrqg5nxP2Py2Lq7gt+/tyzsckREaiWaZwp7gbvcPQ8YAtxiZnnA94F33b0H8G7wGOBcoEdwux54LIq11Ys+ndL45sAujJ9eSOGmnWGXIyLylaIWCu6+zt1nB/dLgcVAZ+B8YHzwtPHABcH984FnPeJjoLWZZUarvvryX1/rSXJiAg+8tTjsUkREvlK9jCmYWQ5wIvAJ0MHd1wWH1gMdgvudgVU1XrY6aDvwva43swIzKygujv0pn+1bpXLTiO5MWriB6cs3h12OiMhhRT0UzKwF8DJwh7tvr3nMI1d3HdEVXu4+zt3z3T2/Xbt2dVhp9Fw3vBud0lK5/41FVFXpgjYRiV1RDQUzSyYSCM+7+ytB84Z93ULBz41B+xogq8bLuwRtDV5qciLfG9WLhWu38/Ls1WGXIyJySNGcfWTAU8Bidx9b49BE4Mrg/pXA32q0XxHMQhoCbKvRzdTgnde/E/2zWvPrSUvYWb437HJERA4qmmcKQ4HLgTPMbG5wGw38AjjbzJYBZwWPAd4EVgCfA08CN0extnqXkGD8aExvNpaW88S0FWGXIyJyUEnRemN3/xCwQxw+8yDPd+CWaNUTCwZ2zeDr/TIZN2053xqcRWZa07BLEhHZj65ormffH9WLKodfvb0k7FJERL5EoVDPsjKace2wXF6ds4ZPV20NuxwRkf0oFEJw88jutG3RhJ++rj0XRCS2KBRC0DI1mTvP7klB0RbenL8+7HJERKopFEJyyaAsenVsyQNvLWaHpqiKSIxQKIQkMcH4yXl9WLetjJufn63NeEQkJigUQnRytzb8/Bt9mba0mHtema/xBREJXdSuU5DauWRQNmu3lvHQu8vo1Lopd559fNgliUgcUyjEgDvO6sG6bbt5+N1ldEpL5dLB2WGXJCJxSqEQA8yMn33jBDZsL+cHry2gQ6tUTu/VPuyyRCQOaUwhRiQnJvDoZSfRO7MlNz8/m3mrt4ZdkojEIYVCDGmeksTTVw2iTYsmXPPMTFZu3hV2SSISZxQKMaZ9y1TGXzOYvVXOlX+aQcnOPWGXJCJxRKEQg7q3a8FTV+azduturh0/k917KsMuSUTihEIhRg3smsFDlw5g7qqt3P7CHCq1jaeI1AOFQgwb1TeTH4/JY/KiDdw3caEubhORqNOU1Bh31dBc1m0r44lpK+jUuik3jewedkki0ogpFBqAu0f1Yu22Mn759mdkpqVywYmdwy5JRBophUIDkJBg/Oab/SguLeO/X/qU9i1TOPW4tmGXJSKNkMYUGoiUpESeuDyf3LbNueG5WSxetz3skkSkEYpaKJjZ02a20cwW1Gh70czmBrdCM5sbtOeY2e4axx6PVl0NWVrTZJ65ejDNU5K4+k8zWbt1d9gliUgjE80zhWeAUTUb3P0Sdx/g7gOAl4FXahxevu+Yu98YxboatE6tm/LMNYPYWb6Xq/40g227K8IuSUQakaiFgrtPA0oOdszMDLgYmBCtz2/MenVsxROXD+SLTTu54bkCyvfq4jYRqRthjSmcBmxw92U12nLNbI6ZTTWz0w71QjO73swKzKyguLg4+pXGqFOPa8tvvtmfj1eU8F9/nUeVLm4TkToQ1uyjb7H/WcI6INvdN5vZQOA1M+vj7l8aTXX3ccA4gPz8/Lj+TXj+gM6s3RqZqtopLZV7RvcOuyQRaeDqPRTMLAm4EBi4r83dy4Hy4P4sM1sOHA8U1Hd9Dc2NI7qxdutunpi2gsy0VK4amht2SSLSgIVxpnAW8Jm7r97XYGbtgBJ3rzSzbkAPYEUItTU4ZsZ95/Vh/fYyfvL6IjqmpTKqb2bYZYlIAxXNKakTgOlATzNbbWbXBocu5csDzMOBecEU1ZeAG939oIPU8mWJCcbDl57IgKzW3P7CXAoK9dWJyNGxhrzIWn5+vhcUqIdpn5Kde7josY/YsmsPL990Kt3btQi7JBGJQWY2y93zD3ZMVzQ3IhnNmzD+6sEkJRhXPj2DjaVlYZckIg2MQqGRyW7TjKeuHMTmHXu45pmZ7CzfG3ZJItKAKBQaof5ZrXn0spNYvK6Um5+fTUVlVdgliUgDoVBopE7v1Z6fXdCXqUuL+cGr87VBj4jUipbObsQuHZzN2m1lPPzuMjq1bsodZx0fdkkiEuMUCo3cd8/qwdqtu3nwnWVkpqVyyaDssEsSkRimUGjkzIwHLjyBjaXl3PvqAtq3SuX0nu3DLktEYlStxhTMrLmZJQT3jzez88wsObqlSV1JTkzg0ctOolfHltzy/Gzmr94WdkkiEqNqO9A8DUg1s87AZOByIvslSAPRIiWJP101iPRmTbj6mZmsKtkVdkkiEoNqGwrm7ruILGT3qLt/E+gTvbIkGtq3SmX8NYOpqKziyqdnsGXnnrBLEpEYU+tQMLNTgMuAN4K2xOiUJNF0XPsW/PHKfFZv3c2142dSVqENekTkX2obCncA9wCvuvvCYCXTf0StKomqQTkZPHTJAOas2srtL8yhUhv0iEigVqHg7lPd/Tx3/2Uw4LzJ3W+Lcm0SReeekMmPxuQxaeEG/vfvC3Vxm4gAtZ999Gcza2VmzYEFwCIz++/olibRdvXQXK47LZfx04v4wz8+VzCISK27j/KCrTEvAN4CconMQJIG7p5ze3Ne/078ZvJSrnh6hmYlicS52oZCcnBdwgXARHevAPRnZSOQkGA8eMkAfnp+H2YXbeGcB6fx9IdfaJxBJE7VNhSeAAqB5sA0M+sKbI9WUVK/EhKMy0/JYfKdIzg5N4P/fX0R//74RyzbUBp2aSJSz4565zUzS3L3UBfr185rdc/dmfjpWu6buJAd5Xu55fTjuHnkcTRJ0oK6Io3FMe+8ZmZpZjbWzAqC22+JnDVII2NmnD+gM+/cOYLRJ2Ty4DvL+Lfff8iclVvCLk1E6kFt//x7GigFLg5u24E/RasoCV+bFik8dOmJPH1VPtvLKrjwsY/46euL2LVHO7mJNGa1DYXu7v5jd18R3H4CdDvcC8zsaTPbaGYLarTdZ2ZrzGxucBtd49g9Zva5mS0xs3OO7p8jde2MXh2Y/N3hXHZyNk99+AXnPDiND5dtCrssEYmS2obCbjMbtu+BmQ0Fdn/Fa54BRh2k/XfuPiC4vRm8Xx5wKZH1lEYBj5qZltGIES1Tk7n/ghP4yw2nkJyQwHee+oT//uunbNtVEXZpIlLHahsKNwJ/MLNCMysEHgFuONwL3H0aUFLL9z8feMHdy939C+BzYHAtXyv1ZHBuBm/efho3j+zOK3PWcObYqbw1f13YZYlIHartMhefunt/oB/Qz91PBM44ys+81czmBd1L6UFbZ2BVjeesDtokxqQmJ/K9Ub2YeOtQOqalcNPzs7nhuQI2bi8LuzQRqQNHNM/Q3bcHVzYD3HkUn/cY0B0YAKwDfnukb2Bm1++bBVVcXHwUJUhd6NMpjdduHso95/bi/SXFnDl2Ki/OXKmlMkQauGOZfG5H+gJ33+Dule5eBTzJv7qI1gBZNZ7aJWg72HuMc/d8d89v167dkZYgdSgpMYEbRnTn7TuGk5fZirtfns9lf/yEos07wy5NRI7SsYTCEf9JaGaZNR5+g8jiegATgUvNLMXMcoEewIxjqE3qUW7b5ky4bgg//8YJzF+9jXMenMaT01awt7Iq7NJE5AglHe6gmZVy8F/+BjT9itdOAEYCbc1sNfBjYKSZDQjes5BgsDrYo+EvwCJgL3CLu2v3lwYkIcH49snZnNGrPT98bQE/e3Mxf5+3ll9e1I/ema3CLk9Eaumol7mIBVrmIja5O2/MX8d9ExeydVcFN43szq1nHEdKkmYZi8SCY17mQuRImBlj+nViyndHcN6ATvz+vc/5+sMfMquotjOURSQsCgWJmvTmTRh78QDGXzOY3Xsq+ffHp/Pjvy1gR7mWyhCJVQoFiboRx7dj8neHc+UpOTz7cRHn/G4a7y/ZGHZZInIQCgWpF81TkrjvvD68dOOpNG2SyFV/msmdL85ly849YZcmIjUoFKReDeyazhu3DeO2M3sw8dO1nDV2KhM/XauL3kRihEJB6l1KUiJ3nn08r982jC7pTbltwhz+Y3wB67Z91RqLIhJtCgUJTa+OrXjl5qH88Ou9+efyTZw9dhrPfVxElfaHFgmNQkFClZhg/Mdp3Zh8xwgGZLXmf15bwEWPf8TiddoCXCQMCgWJCdltmvHctYP53SX9Wbl5F2N+/yE/f3OxdnoTqWcKBYkZZsY3TuzCu3eN4OL8LoybtoKzx07jnUUbwi5NJG4oFCTmtG7WhAcu7MdLN55Ci5Qk/uPZAm54TgPRIvVBoSAxKz8ng9dvG8bdo3oxdWkxZ/12Kk99+IVWXxWJIoWCxLTkxARuGtmdKd8dweDcDH76+iIuePSfzFu9NezSRBolhYI0CFkZzXj6qkE8etlJbNxezgV/+Cf3TVxIaVlF2KWJNCoKBWkwzIzRJ2Tyzl0juHxIV8ZPL+SssVN5c/46XREtUkcUCtLgtEpN5ifn9+W1m4fStkUKNz8/m2uemcmqkl1hlybS4CkUpMHqn9Wav90ylP8Zk8cnX5Rw9u+m8vjU5VRoIFrkqCkUpEFLSkzg2mG5vHPnCIb3aMcv3vqMf/u9NvQROVoKBWkUOrVuyrgr8nnyiny2767gosemc88r89m2SwPRIkdCoSCNytl5HZhy5wiuOy2XvxSs4syx7/PanDUaiBapJYWCNDrNU5L4wdfz+Putw+iS3ow7XpzL5U/N4ItNO8MuTSTmRS0UzOxpM9toZgtqtP3azD4zs3lm9qqZtQ7ac8xst5nNDW6PR6suiR95nVrx8k2n8tML+vLp6q2c8+A0HnpnGeV7K8MuTSRmRfNM4Rlg1AFtU4C+7t4PWArcU+PYcncfENxujGJdEkcSE4zLh3Tl3TtH8LW8DvzunaWc+9AHTF++OezSRGJS1ELB3acBJQe0TXb3fWshfwx0idbni9TUvlUqj3z7JMZfM5i9lc63nvyYu/7yKSXaI1pkP2GOKVwDvFXjca6ZzTGzqWZ22qFeZGbXm1mBmRUUFxdHv0ppVEYc347J3x3OLad3Z+Knazjjt+/zl5mrNBAtErBo/s9gZjnA6+7e94D2HwD5wIXu7maWArRw981mNhB4Dejj7ofdfis/P98LCgqiU7w0ess2lPKDVxcwo7CEwTkZ/OwbfenRoWXYZYlEnZnNcvf8gx2r9zMFM7sKGANc5kEiuXu5u28O7s8ClgPH13dtEl96dGjJC9cP4VcX9WPpxlJGP/wBv570GWUVGoiW+FWvoWBmo4DvAee5+64a7e3MLDG43w3oAayoz9okPiUkGBcPyuLdO0dwXv/O/OEfy/na76bx5vx1VFapS0niTzSnpE4ApgM9zWy1mV0LPAK0BKYcMPV0ODDPzOYCLwE3urvWKZB606ZFCr+9uD8TrhtCk6QEbn5+NmeNncqEGSs1hVXiSlTHFKJNYwoSDZVVzqSF63ns/eXMX7ONdi1TuHZYLt8+OZtWqclhlydyzA43pqBQEDkEd+ej5Zt5fOpyPli2iZYpSVw2pCvXDMuhfcvUsMsTOWoKBZFjNH/1Nh6ftpy35q8jKTGBi07qwg3Du5HTtnnYpYkcMYWCSB0p3LSTcR+s4KVZq6morGJ030xuHNGdE7qkhV2aSK0pFETq2MbSMp75ZyHPfVxEadlehh7XhptGHMfQ49pgZmGXJ3JYCgWRKCktq+DPn6zkqQ+/YGNpOX07t+LGEd05t28miQkKB4lNCgWRKCvfW8mrs9cwbtoKVmzaSU6bZlw3vBsXndSF1OTEsMsT2Y9CQaSeVFY5Uxat57GpK/h01Vbatkjh6qE5fGdIV9KaajqrxAaFgkg9c3emr9jM41NXMG1pMS1Skrjs5GyuGZZLh1aazirhUiiIhGjBmm08MW0Fb8xbS1JCAhee1Jnrh3ejW7sWYZcmcUqhIBIDVm7exZMfrOAvBavYU1nFOXkduXFkdwZktQ67NIkzCgWRGLJpRznP/LOQZ6cXsr1sL6d0a8ONI7szvEdbTWeVeqFQEIlBO8r3MiGYzrp+exl5ma24cWR3RvftSFJimPtfSWOnUBCJYXv2VvHa3DU8PnU5K4p3kp3RjOtOy+Wb+VmazipRoVAQaQCqqpwpizfw2PvLmbtqK22aN+H0Xu0ZlJPOoJwMcts2V/eS1InDhUJSfRcjIgeXkGCc06cjX8vrwCdflPDs9ELe+2wjL81aDUDbFk3I75rBoNwMBudk0DuzpbqZpM4pFERijJkxpFsbhnRrg7uzvHgHMwu3MPOLEmYWlfD2wvUANG+SyEld04OgSOfErHSaNlF3kxwbdR+JNDDrtu1mZuEWCgpLmPFFCUs2lOIOSQlG385pDM7NYFBOBvld00lv3iTsciUGaUxBpBHbtquC2Su3MKOwhJlflDBv9Tb2VFYB0KN9CwblZlSPS3RJbxZytRILFAoicaSsopJ5q7cxs7CEmYUlzCrcQmn5XgA6paWSn5NRHRTHt29JglZzjTsaaBaJI6nJiQzOzWBwbgYQWaRvyfpSZhaWMKOwhI9XbGbip2sBSGuaTH7XdPJzMhicm07fzmmkJGlcIp5FNRTM7GlgDLDR3fsGbRnAi0AOUAhc7O5bLDLX7iFgNLALuMrdZ0ezPpF4kJhg5HVqRV6nVlx5ag7uzqqS3cwoLImMSxSW8O5nGwFISUqgf1ZrBudkkJ+TzsCu6bRM1equ8SSq3UdmNhzYATxbIxR+BZS4+y/M7PtAurvfbWajgf8kEgonAw+5+8mHe391H4nUjc07yiMznIIup4Vrt1NZ5SQY9OmUxjfzu3DhSV1okaLOhcYg1DEFM8sBXq8RCkuAke6+zswygffdvaeZPRHcn3Dg8w713goFkejYWb6XOSu3MrOwhPc+28j8NdtokZLEvw/swuWndKW7Vnht0GJtTKFDjV/064EOwf3OwKoaz1sdtO0XCmZ2PXA9QHZ2dnQrFYlTzVOSGNajLcN6tOW7Zx/PnJVbeHZ6Ec9/UsQzHxVyWo+2XHlKDqf3aq9tRxuZUM8F3d3N7IhOVdx9HDAOImcKUSlMRPZzYnY6J2anc+/o3rwwYyXPf7KS/3i2gC7pTbl8SFcuGZRF62a6JqIxCOMa+Q1BtxHBz41B+xogq8bzugRtIhIj2rVM4T/P7MEHd5/Oo5edRKfWTXngrc84+efvcvdL81i4dlvYJcoxCuNMYSJwJfCL4OffarTfamYvEBlo3na48QQRCU9yYgKjT8hk9AmZLF63nWenF/HanDW8WLCKQTnpXHFKDqP6diRZazM1ONGefTQBGAm0BTYAPwZeA/4CZANFRKaklgRTUh8BRhGZknq1ux92FFkDzSKxY9uuCv46axXPfVxE0eZdtG+ZwrdPzubbJ2fTvqX2pY4luqJZROpNVZUzdWkx46cX8v6SYpITjXP7ZnLlqV05KTtdy3/HgFibfSQijVhCgnF6r/ac3qs9X2zayXPTi/hrwSomfrqWvp1bccUpOZzXv5M2EIpROlMQkajbWb6XV+es4dnphSzdsIP0ZslcMiib7wzJ1iJ9IVD3kYjEBHfn4xUljP+okMmLIvtCnNm7A1eeksPQ49qoa6meqPtIRGKCmXFK9zac0r0Na7bu5s+fFDFhxiqmLNpA93bNufLUHC2nETKdKYhIqMoqKnlj3jrGTy9k3motp1Ef1H0kIg3C3FVbGf9RIW/MW8eeyiotpxElCgURaVCKS8t5ceZK/u/jlazfXkaX9KZ8++RsxpzQiew2Gpg+VgoFEWmQKiqrmLJoA898VMiML0oA6J3ZilF9OjKqb0eO79BCg9NHQaEgIg3eqpJdTFq4nkkL11NQtAV36Na2Oef07cioPh3p1yVNAVFLCgURaVQ2lpYxeeEGJi1cz/Tlm9lb5WSmpXJOcAYxKCdDYxCHoVAQkUZr6649vLt4I28vXM+0pcWU762iTfMmnJ3XgXP6duTU7m207/QBFAoiEhd2lu9l6tJi3l6wnvc+28iO8r20TEnijN7tGdWnIyN6tqNZE10DoVAQkbhTvreSjz7fzNsL1jN50Xq27KogNTmBEce3Y1TfjpzRqwNpTZPDLjMUCgURiWt7K6uYWbiFSQvX8/aC9azfXkZSgnHqcW0Z1acjZ+d1oF3LlLDLrDcKBRGRQFWV8+nqrby9cD2TFqyncPMuzGBQ1wzO6duRc/p0aPSL9CkUREQOwt1ZsqGUtxdEziA+W18KwAmd0xjVNzKTqTEutaFQEBGphcJNOyNdTAvXM2flVgB6tG/BqL4dOadPR/p0atUoroVQKIiIHKH128qYvChyBvHxis1UOXRJb8qZvdqTn5NBfk46mWlNwy7zqCgURESOQcnOPbyzaANvL4wExK49lQB0bt2UgV3Tyc9JZ2DXdHp1bNUgLppTKIiI1JG9lVUsXldKQVEJBYVbKCgqYcP2cgBapCRxYnZr8rtGziQGZLWmeQzuDRFToWBmPYEXazR1A34EtAauA4qD9nvd/c3DvZdCQUTC5u6s3rKbWUVbqoNiyYZS3CExweid2bI6JPK7ZtAxLTXskmMrFPb7cLNEYA1wMnA1sMPdf1Pb1ysURCQWbS+rYHbRlkhQFG5h7qqt7K74V5dTJCDSGdg1g54dW9Z7l1Msb8d5JrDc3Ysaw4i+iAhAq9RkRvZsz8ie7YHIEuCL122noDASFNOXb+Zvc9cC0DIliQHZrRmUk0F+13QGZLcOdSmOsM8UngZmu/sjZnYfcBWwHSgA7nL3LQd5zfXA9QDZ2dkDi4qK6q9gEZE6sK/LaV9306yi/buc8jJbVQ9gR6PLKSa7j8ysCbAW6OPuG8ysA7AJcOCnQKa7X3O491D3kYg0Ftt2VzBnZSQgZhaWMHfVVsoqqoDIVNj8rukMDM4mju9wbF1Osdp9dC6Rs4QNAPt+ApjZk8DrYRUmIlLf0pp+uctp0drtFBRtYVZRCR8t38xrNbqcLh2cxQ++nlfndYQZCt8CJux7YGaZ7r4uePgNYEEoVYmIxIDkxAT6Z7Wmf1Zrrh2WW93lNLOwhIKiLVG7cC6UUDCz5sDZwA01mn9lZgOIdB8VHnBMRCSumRlZGc3IymjGhSd1idrnhBIK7r4TaHNA2+Vh1CIiIv+SEHYBIiISOxQKIiJSTaEgIiLVFAoiIlJNoSAiItUUCiIiUk2hICIi1Rr0JjtmVgwcy4p4bYmstyT6Lg6k7+Nf9F3srzF8H13dvd3BDjToUDhWZlZwqEWh4o2+i/3p+/gXfRf7a+zfh7qPRESkmkJBRESqxXsojAu7gBii72J/+j7+Rd/F/hr19xHXYwoiIrK/eD9TEBGRGhQKIiJSLS5DwcxGmdkSM/vczL4fdj1hMrMsM/uHmS0ys4VmdnvYNYXNzBLNbI6Zxf2WsGbW2sxeMrPPzGyxmZ0Sdk1hMrPvBv+fLDCzCWaWGnZNdS3uQsHMEoE/ENkjOg/4lpnV/UanDcde4C53zwOGALfE+fcBcDuwOOwiYsRDwNvu3gvoTxx/L2bWGbgNyHf3vkAicGm4VdW9uAsFYDDwubuvcPc9wAvA+SHXFBp3X+fus4P7pUT+p+8cblXhMbMuwNeBP4ZdS9jMLA0YDjwF4O573H1rqEWFLwloamZJQDNgbcj11Ll4DIXOwKoaj1cTx78EazKzHOBE4JOQSwnTg8D3gKqQ64gFuUAx8KegO+2Pwf7qccnd1wC/AVYC64Bt7j453KrqXjyGghyEmbUAXgbucPftYdcTBjMbA2x091lh1xIjkoCTgMfc/URgJxC3Y3Bmlk6kVyEX6AQ0N7PvhFtV3YvHUFgDZNV43CVoi1tmlkwkEJ5391fCridEQ4HzzKyQSLfiGWb2f+GWFKrVwGp333fm+BKRkIhXZwFfuHuxu1cArwCnhlxTnYvHUJgJ9DCzXDNrQmSgaGLINYXGzIxIn/Fidx8bdj1hcvd73L2Lu+cQ+e/iPXdvdH8J1pa7rwdWmVnPoOlMYFGIJYVtJTDEzJoF/9+cSSMceE8Ku4D65u57zexWYBKR2QNPu/vCkMsK01DgcmC+mc0N2u519zfDK0liyH8Czwd/QK0Arg65ntC4+ydm9hIwm8isvTk0wiUvtMyFiIhUi8fuIxEROQSFgoiIVFMoiIhINYWCiIhUUyiIiEg1hYLIVzCzSjObW+NWZ1f1mlmOmS2oq/cTOVZxd52CyFHY7e4Dwi5CpD7oTEHkKJlZoZn9yszmm9kMMzsuaM8xs/fMbJ6ZvWtm2UF7BzN71cw+DW77lkhINLMng3X6J5tZ09D+URL3FAoiX63pAd1Hl9Q4ts3dTwAeIbLCKsDvgfHu3g94Hng4aH8YmOru/YmsIbTvSvoewB/cvQ+wFbgoqv8akcPQFc0iX8HMdrh7i4O0FwJnuPuKYFHB9e7exsw2AZnuXhG0r3P3tmZWDHRx9/Ia75EDTHH3HsHju4Fkd7+/Hv5pIl+iMwWRY+OHuH8kymvcr0RjfRIihYLIsbmkxs/pwf2P+Nc2jZcBHwT33wVugup9oNPqq0iR2tJfJCJfrWmNFWQhsmfxvmmp6WY2j8hf+98K2v6TyG5l/01k57J9K4veDowzs2uJnBHcRGQHL5GYoTEFkaMUjCnku/umsGsRqSvqPhIRkWo6UxARkWo6UxARkWoKBRERqaZQEBGRagoFERGpplAQEZFq/w8ISrewv9PBAAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(0,epochs),[loss for loss in list(epoch_loss.values())])\n",
    "plt.title('Loss graph')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}