21_seq2seq_generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 빅데이터 활용 AI 설계\n",
    "# Seq2seq Eng-Fra Translater\n",
    "- 제너레이터 적용하여 전체 문장 로딩"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "import string\n",
    "from string import digits\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 영어-불어 번역 파일 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.read_csv('fra.txt', delimiter='\\t', names=['eng', 'fr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Va !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Run!</td>\n",
       "      <td>Cours !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Run!</td>\n",
       "      <td>Courez !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Wow!</td>\n",
       "      <td>Ça alors !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fire!</td>\n",
       "      <td>Au feu !</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     eng          fr\n",
       "0    Go.        Va !\n",
       "1   Run!     Cours !\n",
       "2   Run!    Courez !\n",
       "3   Wow!  Ça alors !\n",
       "4  Fire!    Au feu !"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149856</th>\n",
       "      <td>A carbon footprint is the amount of carbon dio...</td>\n",
       "      <td>Une empreinte carbone est la somme de pollutio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149857</th>\n",
       "      <td>Death is something that we're often discourage...</td>\n",
       "      <td>La mort est une chose qu'on nous décourage sou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149858</th>\n",
       "      <td>Since there are usually multiple websites on a...</td>\n",
       "      <td>Puisqu'il y a de multiples sites web sur chaqu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149859</th>\n",
       "      <td>If someone who doesn't know your background sa...</td>\n",
       "      <td>Si quelqu'un qui ne connaît pas vos antécédent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149860</th>\n",
       "      <td>It may be impossible to get a completely error...</td>\n",
       "      <td>Il est peut-être impossible d'obtenir un Corpu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      eng  \\\n",
       "149856  A carbon footprint is the amount of carbon dio...   \n",
       "149857  Death is something that we're often discourage...   \n",
       "149858  Since there are usually multiple websites on a...   \n",
       "149859  If someone who doesn't know your background sa...   \n",
       "149860  It may be impossible to get a completely error...   \n",
       "\n",
       "                                                       fr  \n",
       "149856  Une empreinte carbone est la somme de pollutio...  \n",
       "149857  La mort est une chose qu'on nous décourage sou...  \n",
       "149858  Puisqu'il y a de multiples sites web sur chaqu...  \n",
       "149859  Si quelqu'un qui ne connaît pas vos antécédent...  \n",
       "149860  Il est peut-être impossible d'obtenir un Corpu...  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(149861, 2)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 전체 문장 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(149861, 2)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### 전체 문장 사용시 CPU 에러 발생!!! (내부 텐서 사용시 단어사전 차원이 커서 발생하는 듯)\n",
    "\n",
    "#lines = df[:10000].copy()\n",
    "lines = df.copy()\n",
    "lines.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149856</th>\n",
       "      <td>A carbon footprint is the amount of carbon dio...</td>\n",
       "      <td>Une empreinte carbone est la somme de pollutio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149857</th>\n",
       "      <td>Death is something that we're often discourage...</td>\n",
       "      <td>La mort est une chose qu'on nous décourage sou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149858</th>\n",
       "      <td>Since there are usually multiple websites on a...</td>\n",
       "      <td>Puisqu'il y a de multiples sites web sur chaqu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149859</th>\n",
       "      <td>If someone who doesn't know your background sa...</td>\n",
       "      <td>Si quelqu'un qui ne connaît pas vos antécédent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149860</th>\n",
       "      <td>It may be impossible to get a completely error...</td>\n",
       "      <td>Il est peut-être impossible d'obtenir un Corpu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      eng  \\\n",
       "149856  A carbon footprint is the amount of carbon dio...   \n",
       "149857  Death is something that we're often discourage...   \n",
       "149858  Since there are usually multiple websites on a...   \n",
       "149859  If someone who doesn't know your background sa...   \n",
       "149860  It may be impossible to get a completely error...   \n",
       "\n",
       "                                                       fr  \n",
       "149856  Une empreinte carbone est la somme de pollutio...  \n",
       "149857  La mort est une chose qu'on nous décourage sou...  \n",
       "149858  Puisqu'il y a de multiples sites web sur chaqu...  \n",
       "149859  Si quelqu'un qui ne connaît pas vos antécédent...  \n",
       "149860  Il est peut-être impossible d'obtenir un Corpu...  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터 정리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines.eng=lines.eng.apply(lambda x: x.lower())\n",
    "lines.fr=lines.fr.apply(lambda x: x.lower())\n",
    "\n",
    "lines.eng=lines.eng.apply(lambda x: re.sub(\"'\", '', x)).apply(lambda x: re.sub(\",\", ' COMMA', x))\n",
    "lines.fr=lines.fr.apply(lambda x: re.sub(\"'\", '', x)).apply(lambda x: re.sub(\",\", ' COMMA', x))\n",
    "\n",
    "exclude = set(string.punctuation) # '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'\n",
    "lines.eng=lines.eng.apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n",
    "lines.fr=lines.fr.apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n",
    "\n",
    "# 숫자를 지운다\n",
    "remove_digits = str.maketrans('', '', digits)\n",
    "lines.eng=lines.eng.apply(lambda x: x.translate(remove_digits))\n",
    "lines.fr=lines.fr.apply(lambda x: x.translate(remove_digits))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 불어 문장에 태그 달기\n",
    "- START_ 와 _END"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines.fr = lines.fr.apply(lambda x : 'START_ '+ x + ' _END')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>go</td>\n",
       "      <td>START_ va  _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>run</td>\n",
       "      <td>START_ cours  _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>run</td>\n",
       "      <td>START_ courez  _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>wow</td>\n",
       "      <td>START_ ça alors  _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fire</td>\n",
       "      <td>START_ au feu  _END</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    eng                     fr\n",
       "0    go        START_ va  _END\n",
       "1   run     START_ cours  _END\n",
       "2   run    START_ courez  _END\n",
       "3   wow  START_ ça alors  _END\n",
       "4  fire    START_ au feu  _END"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 단어 목록 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng_words=set()\n",
    "for eng in lines.eng:\n",
    "    for word in eng.split():\n",
    "        eng_words.add(word)\n",
    "    \n",
    "fra_words=set()\n",
    "for fr in lines.fr:\n",
    "    for word in fr.split():\n",
    "        fra_words.add(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13799, 27980)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eng_words = sorted(list(eng_words))\n",
    "fra_words = sorted(list(fra_words))\n",
    "\n",
    "len(eng_words), len(fra_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['COMMA',\n",
       " 'a',\n",
       " 'aback',\n",
       " 'abacus',\n",
       " 'abandon',\n",
       " 'abandoned',\n",
       " 'abandoning',\n",
       " 'abated',\n",
       " 'abdominal',\n",
       " 'abducted']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['COMMA',\n",
       " 'COMMAenfila',\n",
       " 'START_',\n",
       " '_END',\n",
       " 'a',\n",
       " 'abaissa',\n",
       " 'abaisse',\n",
       " 'abaisser',\n",
       " 'abaissez',\n",
       " 'abaissé']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(eng_words[:10], fra_words[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 단어 사전 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng_w2i = { word: i for i, word in enumerate(eng_words) }\n",
    "fra_w2i = { word: i for i, word in enumerate(fra_words) }\n",
    "\n",
    "eng_i2w = { i: word for word, i in eng_w2i.items() }\n",
    "fra_i2w = { i: word for word, i in fra_w2i.items() }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13799, 27980)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(eng_i2w), len(fra_i2w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 최대 문장 길이"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49, 58)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eng_sen_max = max([len(sen.split()) for sen in lines.eng])\n",
    "fra_sen_max = max([len(sen.split()) for sen in lines.fr])\n",
    "\n",
    "eng_sen_max, fra_sen_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 인코더  생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Embedding, LSTM, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 50\n",
    "\n",
    "enc_inputs = Input((eng_sen_max,))\n",
    "enc_embed = Embedding(len(eng_words),embedding_size)(enc_inputs)\n",
    "\n",
    "enc_lstm = LSTM(50, return_state=True)\n",
    "enc_output, state_h, state_c = enc_lstm(enc_embed)\n",
    "\n",
    "enc_states = [state_h, state_c]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 디코더 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "dec_inputs = Input((fra_sen_max,))\n",
    "dec_embed = Embedding(len(fra_words), embedding_size)(dec_inputs)\n",
    "\n",
    "dec_lstm = LSTM(50, return_sequences=True)\n",
    "dec_outputs = dec_lstm(dec_embed, initial_state=enc_states)\n",
    "\n",
    "dec_outputs = Dense(len(fra_words), activation='softmax')(dec_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model([enc_inputs, dec_inputs], dec_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 49)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            (None, 58)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 49, 50)       689950      input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 58, 50)       1399000     input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_5 (LSTM)                   [(None, 50), (None,  20200       embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_6 (LSTM)                   (None, 58, 50)       20200       embedding_6[0][0]                \n",
      "                                                                 lstm_5[0][1]                     \n",
      "                                                                 lstm_5[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 58, 27980)    1426980     lstm_6[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 3,556,330\n",
      "Trainable params: 3,556,330\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 입출력 제너레이터"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'encoder_input_data = np.zeros([len(lines), eng_sen_max])\\ndecoder_input_data = np.zeros([len(lines), fra_sen_max])\\ndecoder_output_data = np.zeros([len(lines), fra_sen_max, len(fra_words)])'"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''encoder_input_data = np.zeros([len(lines), eng_sen_max])\n",
    "decoder_input_data = np.zeros([len(lines), fra_sen_max])\n",
    "decoder_output_data = np.zeros([len(lines), fra_sen_max, len(fra_words)])'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'for line, (eng,fra) in enumerate(zip(lines.eng, lines.fr)):\\n    for t, word in enumerate(eng.split()):\\n        encoder_input_data[line,t] = eng_w2i[word]\\n        \\n    for t, word in enumerate(fra.split()):\\n        decoder_input_data[line,t] = fra_w2i[word]\\n        if t>0:\\n            decoder_output_data[line,t-1,fra_w2i[word]] = 1 # START_ 가 빠지고 하나씩 당겨짐'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''for line, (eng,fra) in enumerate(zip(lines.eng, lines.fr)):\n",
    "    for t, word in enumerate(eng.split()):\n",
    "        encoder_input_data[line,t] = eng_w2i[word]\n",
    "        \n",
    "    for t, word in enumerate(fra.split()):\n",
    "        decoder_input_data[line,t] = fra_w2i[word]\n",
    "        if t>0:\n",
    "            decoder_output_data[line,t-1,fra_w2i[word]] = 1 # START_ 가 빠지고 하나씩 당겨짐'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(X = lines.eng, y = lines.fr, batch_size = 128):\n",
    "    while True:\n",
    "        for j in range(0, len(X), batch_size):\n",
    "            \n",
    "            encoder_input_data = np.zeros((batch_size, eng_sen_max),dtype='float32') # (128,34)\n",
    "            decoder_input_data = np.zeros((batch_size, fra_sen_max),dtype='float32') # (128,37)\n",
    "            decoder_output_data = np.zeros((batch_size, fra_sen_max, len(fra_words)),dtype='float32')\n",
    "            \n",
    "            for i, (eng, fra) in enumerate(zip(X[j:j+batch_size], y[j:j+batch_size])):\n",
    "                for t, word in enumerate(eng.split()):\n",
    "                    encoder_input_data[i, t] = eng_w2i[word]\n",
    "                    \n",
    "                for t, word in enumerate(fra.split()):\n",
    "                    decoder_input_data[i, t] = fra_w2i[word]\n",
    "                    if t>0: # 앞부분 'START_' 제외\n",
    "                        decoder_output_data[i, t - 1, fra_w2i[word]] = 1\n",
    "                        \n",
    "            yield([encoder_input_data, decoder_input_data], decoder_output_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 학습 시키기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'h = model.fit([encoder_input_data, decoder_input_data], decoder_output_data,\\n              batch_size=128, epochs=10, validation_split=0.1)'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''h = model.fit([encoder_input_data, decoder_input_data], decoder_output_data,\n",
    "              batch_size=128, epochs=10, validation_split=0.1)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[7424,27980] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[{{node dense_3/MatMul}} = MatMul[T=DT_FLOAT, _class=[\"loc:@training_2/RMSprop/gradients/dense_3/MatMul_grad/MatMul\"], transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](dense_3/Reshape, dense_3/kernel/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m----------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mResourceExhaustedError\u001b[0m               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-77-8d7e1e353a7d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m model.fit_generator(generator = generate_batch(lines.eng, lines.fr, batch_size = 128),\n\u001b[0;32m      2\u001b[0m                     \u001b[0msteps_per_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlines\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m//\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m                     epochs=1)\n\u001b[0m",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[0;32m     90\u001b[0m                               'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 91\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   1416\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1417\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1418\u001b[1;33m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[0;32m   1419\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1420\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\engine\\training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m    215\u001b[0m                 outs = model.train_on_batch(x, y,\n\u001b[0;32m    216\u001b[0m                                             \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 217\u001b[1;33m                                             class_weight=class_weight)\n\u001b[0m\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    219\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[0;32m   1215\u001b[0m             \u001b[0mins\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1216\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1217\u001b[1;33m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1218\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2713\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2714\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2715\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2716\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2717\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2674\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2675\u001b[1;33m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2676\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[0;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1439\u001b[1;33m               run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1440\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[0;32m    526\u001b[0m             \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    527\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 528\u001b[1;33m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[0;32m    529\u001b[0m     \u001b[1;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    530\u001b[0m     \u001b[1;31m# as there is a reference to status from this from the traceback due to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[7424,27980] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[{{node dense_3/MatMul}} = MatMul[T=DT_FLOAT, _class=[\"loc:@training_2/RMSprop/gradients/dense_3/MatMul_grad/MatMul\"], transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](dense_3/Reshape, dense_3/kernel/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(generator = generate_batch(lines.eng, lines.fr, batch_size = 128),\n",
    "                    steps_per_epoch = len(lines)//128,\n",
    "                    epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x2aa783ccc18>"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG9pJREFUeJzt3X+UVOWd5/H3t3/RQDe/fzR00yCBBBRWmLSObk6IMbNqsv7YGDfBqBk9rpyo8dcZXcfkJOOa5GQm2aNxT1wddjRqQhIYdXdYdTBrzIqcOKwNNiKgDIP8KEDobqCRYNPQ/d0/nq6p6urqruruaqr71ud1znPuraqnbj9V4ufe+9Rzn2vujoiIREtRvhsgIiK5p3AXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIyhjuZlZuZv/PzDaZ2RYz+y9p6owws5VmtsPM1pvZrMForIiIZCebI/eTwMXufi6wCLjMzC5IqXMzcMTd5wCPAH+T22aKiEhfZAx3D453PiztLKlXPl0FPNO5/hzwBTOznLVSRET6pCSbSmZWDGwA5gCPufv6lCrVwF4Adz9tZi3ARKApZTvLgGUAo0eP/vS8efMG1noRkQKzYcOGJnefnKleVuHu7u3AIjMbB/xPM1vg7u8mVUl3lN5tXgN3Xw4sB6irq/P6+vps/ryIiHQys93Z1OvTaBl3Pwr8X+CylJdiwIzOP1wCjAUO92XbIiKSO9mMlpncecSOmY0E/gx4L6XaauDPO9evAV5zzUgmIpI32XTLTAOe6ex3LwJWufuLZvYQUO/uq4EngV+Y2Q7CEfvSQWuxiIhklDHc3f0dYHGa57+XtN4K/MfcNk1EoujUqVPEYjFaW1vz3ZQhrby8nJqaGkpLS/v1/qx+UBURyZVYLEZlZSWzZs1CI6bTc3eam5uJxWKcddZZ/dqGph8QkTOqtbWViRMnKth7YWZMnDhxQGc3CncROeMU7JkN9DtSuIuIRJDCXUQKTkVFRb6bMOgU7iIiEaRwF5GC5e7cd999LFiwgIULF7Jy5UoADhw4wJIlS1i0aBELFizgjTfeoL29nRtvvPFf6z7yyCN5bn3vNBRSRPLm7ruhoSG321y0CH760+zqvvDCCzQ0NLBp0yaampo477zzWLJkCb/61a+49NJL+c53vkN7ezsnTpygoaGBffv28e67YVqto0eP5rbhOaYjdxEpWOvWrePaa6+luLiYqVOn8rnPfY633nqL8847j5///Oc8+OCDbN68mcrKSmbPns3OnTu54447WLNmDWPGjMl383ulI3cRyZtsj7AHS09TYC1ZsoS1a9fy0ksvccMNN3DffffxjW98g02bNvHKK6/w2GOPsWrVKp566qkz3OLs6chdRArWkiVLWLlyJe3t7TQ2NrJ27VrOP/98du/ezZQpU7jlllu4+eab2bhxI01NTXR0dPCVr3yF73//+2zcuDHfze+VjtxFpGB9+ctf5s033+Tcc8/FzPjxj39MVVUVzzzzDD/5yU8oLS2loqKCZ599ln379nHTTTfR0dEBwI9+9KM8t753lq+ZeXWzDpHCtG3bNubPn5/vZgwL6b4rM9vg7nWZ3qtuGRGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuItKL3uZ+37VrFwsWLDiDrcmewl1EJII0/YCI5E8e5vy9//77mTlzJrfddhsADz74IGbG2rVrOXLkCKdOneIHP/gBV111VZ/+bGtrK7feeiv19fWUlJTw8MMP8/nPf54tW7Zw00030dbWRkdHB88//zzTp0/nq1/9KrFYjPb2dr773e/yta99bUAfO5XCXUQKytKlS7n77rv/NdxXrVrFmjVruOeeexgzZgxNTU1ccMEFXHnllX26SfVjjz0GwObNm3nvvfe45JJL2L59O0888QR33XUX1113HW1tbbS3t/Pyyy8zffp0XnrpJQBaWlpy/jkV7iKSP3mY83fx4sUcOnSI/fv309jYyPjx45k2bRr33HMPa9eupaioiH379nHw4EGqqqqy3u66deu44447AJg3bx4zZ85k+/btXHjhhfzwhz8kFotx9dVXM3fuXBYuXMi9997L/fffz+WXX85nP/vZnH9O9bmLSMG55ppreO6551i5ciVLly5lxYoVNDY2smHDBhoaGpg6dSqtra192mZPkzB+/etfZ/Xq1YwcOZJLL72U1157jU9+8pNs2LCBhQsX8sADD/DQQw/l4mN1oSN3ESk4S5cu5ZZbbqGpqYnXX3+dVatWMWXKFEpLS/n973/P7t27+7zNJUuWsGLFCi6++GK2b9/Onj17+NSnPsXOnTuZPXs2d955Jzt37uSdd95h3rx5TJgwgeuvv56KigqefvrpnH/GjOFuZjOAZ4EqoANY7u6PptS5CPgH4IPOp15w99zvikREcuCcc87ho48+orq6mmnTpnHddddxxRVXUFdXx6JFi5g3b16ft3nbbbfxzW9+k4ULF1JSUsLTTz/NiBEjWLlyJb/85S8pLS2lqqqK733ve7z11lvcd999FBUVUVpayuOPP57zz5hxPnczmwZMc/eNZlYJbAD+g7tvTapzEXCvu1+e7R/WfO4ihUnzuWdvUOdzd/cD7r6xc/0jYBtQ3c+2iojIGdCnPnczmwUsBtaneflCM9sE7CccxW8ZcOtERIaAzZs3c8MNN3R5bsSIEaxfny4Kh4asw93MKoDngbvd/VjKyxuBme5+3My+BPwvYG6abSwDlgHU1tb2u9EiMry5e5/GkOfbwoULacj1xVYZDPQWqFkNhTSzUkKwr3D3F9I04pi7H+9cfxkoNbNJaeotd/c6d6+bPHnygBouIsNTeXk5zc3NAw6vKHN3mpubKS8v7/c2shktY8CTwDZ3f7iHOlXAQXd3MzufsNNo7nerRCSyampqiMViNDY25rspQ1p5eTk1NTX9fn823TKfAW4ANptZ/Lzk20AtgLs/AVwD3Gpmp4GPgaWu3bKIpFFaWspZZ52V72ZEXsZwd/d1QK+dY+7+M+BnuWqUiIgMjKYfEBGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIKGX7i3t4N7vlshIjKkleS7AX324ouwdCnMmNF7GTs23y0VEcmb4Rfus2bB7bfDnj2wdy+8+iocOAAdHV3rVVaGkK+t7XkHMHJkXj6CiMhgG37hfu65oSQ7fRr27w9hn65s3AiHDnXf1sSJ6UM/vkOorobS0jPzuUREcmj4hXs6JSUhkGtre67T2gqxWPrw37UL3ngDjh7t+h4zqKrqvfunqgqKiwf144mI9FXGcDezGcCzQBXQASx390dT6hjwKPAl4ARwo7tvzH1zB6C8HObMCaUnx493Df1418/evfDuu/CP/wgnTnR9T0lJOMJP1wUUfzxhQthRiIicIdkcuZ8G/sLdN5pZJbDBzP6Pu29NqvNFYG5n+VPg8c7l8FJRAfPnh5KOOxw50v3IP74TePNN+Pu/h1Onur5v5Mju4Z+6I6ioGPzPJyIFI2O4u/sB4EDn+kdmtg2oBpLD/SrgWXd34J/MbJyZTet8b3SYhaPwCRO69/vHdXTAwYPpj/737oU1a+DDD7sP5xw/vufgr60NZwdlZYP/GUUkEvrU525ms4DFwPqUl6qBvUmPY53PdQl3M1sGLAOo7a1/fDgrKoJp00I5//z0ddraEj8AJ4d/8hnA4cNd32MGU6f2PAKoujr0/+sHYBGhD+FuZhXA88Dd7n4s9eU0b+l2pZG7LweWA9TV1RXulUhlZWFI56xZPdf54x977v7ZsiV9/78ZTJkSgj61TJ+eWB83Tr8BiERcVuFuZqWEYF/h7i+kqRIDZiQ9rgH2D7x5BWz0aJg3L5R0kvv/YzHYt69r2b0b/vAHaG7u/t6RI3sO/niZNk3dQCLDWDajZQx4Etjm7g/3UG018C0z+w3hh9SWyPW3DzXZ9P9DGAK6f3/X4E9+/OabYdnW1v29kydnPgvQSCCRISmbI/fPADcAm82sofO5bwO1AO7+BPAyYRjkDsJQyJty31Tpl/JymD07lJ64hyP81OCPl1gM1q+Hpqb024+H/fjx4Yxj1KiwTF7P9rmyMu0sRHIgm9Ey60jfp55cx4Hbc9UoOcPMYNKkUHo7Czh5Mkz1kBr+8bJrV/it4MSJsPzjH8NEb31RXNw9/Pu6g6isDHMLjR0LY8aE5ahR2mlIQYnGFapyZowYkfmH4FRtbYmwTw791Od6WiavHznSvc7Jk9m1o7g4EfQ9LbN5TaORZJhQuMvgKisLZdy4wdn+6dMh5JN3Bh99BC0tcOxYWCavJz934AC8917icerFZ+mMHJn9TqGysucycqTOJGRQKdxleCspCUE6ZszAt9XamnmHkO65Dz9MPHcsdZRwD4qLw1XJve0AMpXk948YMfDPP1DuYWfb3t69lJaG/0baoZ0xCneRuPLyUKZM6f82OjrCmcOxY2HZ13LwYFgePx6W6UYxpVNamn4HMGpU+rDNpvQU1D2VTDfRKSkJo6smTQozssZL8uPU9fHjNTFfPyncRXKpqCjRVZMLbW3920nEu6YOHAjhmK6MGNH9uZKSnuv3VLJ5T1tbuOq6qSmMzGpuhh07wiis5uaed2JmoUsv004geWcxceLQOJPJM4W7yFBWVpYIrKhyD2cq8dBP3gGkru/fD5s3h/XUK7STVVSk3wmMHh12RvEdUrr1/r7Wl3ojRgz6RYIKdxHJL7NEN1JfRmK1tma3Q2huhn/5l8QOob29+53bzrT774e//utB/RMKdxEZnsrLE1dK91VHR9ffFU6f7n091/XOOy/330cKhbuIFJ6iolAifN1CUb4bICIiuadwFxGJIIW7iEgEDbs+9z174I03ws2IZs4MExKWDLtPISIyuIZdLK5bB9dfn3hcVBR+LJ85MwR+PPSTl5WV+WuviEg+DLtwv/pq2LYtHMHv2RNuOBRfvvkmrFoVRholGzeue+Anr1dVhZ2EiEhUDLtwLy/v/e5z7e1hHqfk4I+v794Na9eGq7KTlZYm7judLvxra8MkfiIiw8WwC/dMiosT1zVceGH6Oi0t4dajyUf98Z3A734XrnBOvYBt8uTugf+JT4R7W9TWarI7ERlaIhfu2YjP67RgQfrXT50KNxZK1/WzbRusWdN1Wovx42HRolAWLw7LefMifX2EiAxxBRnumZSW9n7DIfcwwd327dDQkCiPPx6mu4AwL9CCBYmwX7QoHOVXVJypTyEihcw80xzMg6Surs7r6+vz8rcHy+nTIfDffjuE/dtvh3L4cHjdDObMSQR+fFlVld92i8jwYWYb3L0uYz2F++ByD108yYHf0AAffJCoM3Vq98CfM0cjeESku2zDXd0yg8wMampCueKKxPNHj8KmTYmwb2iAV19NDOMcPTp04yR36yxYEEYLiYhkoiP3IeTkSdi6tWvgNzSEm+pAGAk0f37Xo/xzzw13LhORwqAj92FoxIgQ2IsXJ57r6AhdOMndOr/7HfziF4k6NTVwzjmhnH12YpmLe0aLyPCkcB/iiorCePpPfAKuuSbx/KFDibDfvDkc8b/+emK0DqQP/fnzc3d7TxEZuhTuw9SUKXDJJaHEtbfDrl2wZUsoW7eGZU+hn3yUf/bZCn2RKFGfewFIDv144G/dGi7I+vjjRL2amq6BH18q9EWGjpwNhTSzp4DLgUPu3u2aTjO7CPgHID647wV3fyjTH1a451889JMDf8uW7qFfXZ2+T1+hL3Lm5fIH1aeBnwHP9lLnDXe/PMu2yRBRXJzoz08eptneHqZaSA78LVvgiSfSh3488OfMCXPv1NRo6gWRfMsY7u6+1sxmDX5TZKgoLobZs0NJDv2OjvTdO3/7t11Dv6go3ERl5sz0pbY2jOMXkcGTqx9ULzSzTcB+4F5335KukpktA5YB1NbW5uhPy5lSVNR76O/cmZhaOV7+8If0c+xPmtQ98JMfT5igmTZFBiKrH1Q7j9xf7KHPfQzQ4e7HzexLwKPuPjfTNtXnXjja28M0ysmza6aW5Fk2IUywlhr4yWXaNE3PIIXpjF3E5O7HktZfNrP/bmaT3L1poNuWaCguDjdDmTEj/evu0NycPvR374b16xOTr8XFb7CSLvinTw+TsY0dq6N/KVwDDnczqwIOurub2flAEdA84JZJwTAL3TSTJsGnP52+zvHjPYf/b38LBw6EnUSy8vIQ8unKtGmJ9alTw9XBIlGSMdzN7NfARcAkM4sBfwWUArj7E8A1wK1mdhr4GFjq+Ro8L5FVUZEYjplOW1u4u9aePSHoP/wwlPj6jh3h5upNPZxPjh/fPfTT7QwmTFB3kAwPuohJCsqpU3DwYCL8k0vqTiF5BFBcSUk40k93BpBaNCJIBoMmDhNJo7Q0MQVzb9xDV1Bq6CeH/759sGFDmOcn9Z67EG6qHu9uyqZMnKjuIckdhbtIGmZQWRnK3Axjv9rbQ3dP6o6gqSmU5uaw/OCDsDx6tOdtVVb2bYcwYUI4mxBJpX8WIgNUXBy6aqZODfPrZ3LqVBj9Ew//nkpjY5gKoqkpnEX0ZPz43s8GJkwIdSZMSKyPGqWRRFGncBc5w0pLEzuDbLW2Js4AUs8IkkssFqaCbmzsOhNoqrKyROBnWiavjxunqSWGC4W7yDBQXh7m8qmuzv49J06EwD9yJJwpxJfJ6/Hl/v3w7rvhcUtL79utrOz7jmH8+PA+jTQ6cxTuIhE1alS4yrevM32cPh1+F0jdAfS0c9i2LfH45Mmet1tUFC4sGzcuhP24cX1bLy9XV1JfKNxFpIuSkkSffV99/HHPZwdHjybKkSNh+f77ifXUKShSlZV1D/1sdw6F2J2kcBeRnBk5su/dR3Ftbel3APFl6vrhw2GyuvjzqZPTpRo9OoR8/Oxh7Niu65leGz16eJ05KNxFZEgoKwu3j5wype/vdQ9H/r3tDOLrLS1heegQ/PM/J17PtHMoLg43nc92Z5DutbKy/n03/aFwF5FhzywcWY8e3b+zBvfQpRQP/paWruupy/h6/MyhpQWOHcv8d8rLQ8jfeSc88EDf29kXCncRKXhm4QfoUaPClBL90d4OH33U+84gvsx0YVwuKNxFRHKguDjx4+3MmfluTZieV0REIkbhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiaCM4W5mT5nZITN7t4fXzcz+m5ntMLN3zOxPct9MERHpi2yO3J8GLuvl9S8CczvLMuDxgTdLREQGImO4u/ta4HAvVa4CnvXgn4BxZtbPG1WJiEgu5KLPvRrYm/Q41vlcN2a2zMzqzay+sbExB39aRETSyUW4W5rnPF1Fd1/u7nXuXjd58uQc/GkREUknF+EeA2YkPa4B9udguyIi0k+5CPfVwDc6R81cALS4+4EcbFdERPqpJFMFM/s1cBEwycxiwF8BpQDu/gTwMvAlYAdwArhpsBorIiLZyRju7n5thtcduD1nLRIRkQHTFaoiIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBGUV7mZ2mZm9b2Y7zOwv07x+o5k1mllDZ/lPuW+qiIhkqyRTBTMrBh4D/h0QA94ys9XuvjWl6kp3/9YgtFFERPoomyP384Ed7r7T3duA3wBXDW6zRERkILIJ92pgb9LjWOdzqb5iZu+Y2XNmNiMnrRMRkX7JJtwtzXOe8vh/A7Pc/d8ArwLPpN2Q2TIzqzez+sbGxr61VEREspZNuMeA5CPxGmB/cgV3b3b3k50P/wfw6XQbcvfl7l7n7nWTJ0/uT3tFRCQL2YT7W8BcMzvLzMqApcDq5ApmNi3p4ZXAttw1UURE+irjaBl3P21m3wJeAYqBp9x9i5k9BNS7+2rgTjO7EjgNHAZuHMQ2i4hIBuae2n1+ZtTV1Xl9fX1e/raIyHBlZhvcvS5TPV2hKiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIKyCnczu8zM3jezHWb2l2leH2FmKztfX29ms3LdUBERyV7GcDezYuAx4IvA2cC1ZnZ2SrWbgSPuPgd4BPibXDdURESyl82R+/nADnff6e5twG+Aq1LqXAU807n+HPAFM7PcNVNERPqiJIs61cDepMcx4E97quPup82sBZgINCVXMrNlwLLOh8fN7P3+NBqYlLrtAqfvoyt9Hwn6LrqKwvcxM5tK2YR7uiNw70cd3H05sDyLv9l7g8zq3b1uoNuJCn0fXen7SNB30VUhfR/ZdMvEgBlJj2uA/T3VMbMSYCxwOBcNFBGRvssm3N8C5prZWWZWBiwFVqfUWQ38eef6NcBr7t7tyF1ERM6MjN0ynX3o3wJeAYqBp9x9i5k9BNS7+2rgSeAXZraDcMS+dDAbTQ66diJG30dX+j4S9F10VTDfh+kAW0QkenSFqohIBCncRUQiaNiFe6apEAqJmc0ws9+b2TYz22Jmd+W7TflmZsVm9raZvZjvtuSbmY0zs+fM7L3OfyMX5rtN+WJm93T+P/Kumf3azMrz3abBNqzCPcupEArJaeAv3H0+cAFwe4F/HwB3Advy3Ygh4lFgjbvPA86lQL8XM6sG7gTq3H0BYWDIYA/6yLthFe5kNxVCwXD3A+6+sXP9I8L/vNX5bVX+mFkN8O+Bv8t3W/LNzMYASwgj2XD3Nnc/mt9W5VUJMLLzOpxRdL9WJ3KGW7inmwqhYMMsWedMnIuB9fltSV79FPjPQEe+GzIEzAYagZ93dlP9nZmNznej8sHd9wH/FdgDHABa3P23+W3V4Btu4Z7VNAeFxswqgOeBu939WL7bkw9mdjlwyN035LstQ0QJ8CfA4+6+GPgjUJC/UZnZeMIZ/lnAdGC0mV2f31YNvuEW7tlMhVBQzKyUEOwr3P2FfLcnjz4DXGlmuwjddReb2S/z26S8igExd4+fyT1HCPtC9GfAB+7e6O6ngBeAf5vnNg264Rbu2UyFUDA6p1V+Etjm7g/nuz355O4PuHuNu88i/Lt4zd0jf3TWE3f/ENhrZp/qfOoLwNY8Nimf9gAXmNmozv9nvkAB/LiczayQQ0ZPUyHkuVn59BngBmCzmTV0Pvdtd385j22SoeMOYEXngdBO4KY8tycv3H29mT0HbCSMMHubApiGQNMPiIhE0HDrlhERkSwo3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEfT/AQ/K8sIu/XMuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(h.history['loss'], 'b-')\n",
    "plt.plot(h.history['val_loss'], 'r-')\n",
    "plt.ylim(0,3)\n",
    "plt.legend(['loss', 'val_loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 9000 samples, validate on 1000 samples\n",
      "Epoch 1/120\n",
      "9000/9000 [==============================] - 191s 21ms/step - loss: 1.2554 - acc: 0.1240 - val_loss: 1.8178 - val_acc: 0.1112\n",
      "Epoch 2/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 1.2295 - acc: 0.1268 - val_loss: 1.8049 - val_acc: 0.1108\n",
      "Epoch 3/120\n",
      "9000/9000 [==============================] - 159s 18ms/step - loss: 1.2050 - acc: 0.1295 - val_loss: 1.7932 - val_acc: 0.1116\n",
      "Epoch 4/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 1.1811 - acc: 0.1321 - val_loss: 1.7891 - val_acc: 0.1135\n",
      "Epoch 5/120\n",
      "9000/9000 [==============================] - 154s 17ms/step - loss: 1.1580 - acc: 0.1353 - val_loss: 1.7669 - val_acc: 0.1174\n",
      "Epoch 6/120\n",
      "9000/9000 [==============================] - 162s 18ms/step - loss: 1.1362 - acc: 0.1376 - val_loss: 1.7494 - val_acc: 0.1208\n",
      "Epoch 7/120\n",
      "9000/9000 [==============================] - 156s 17ms/step - loss: 1.1156 - acc: 0.1418 - val_loss: 1.7430 - val_acc: 0.1232\n",
      "Epoch 8/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 1.0956 - acc: 0.1438 - val_loss: 1.7237 - val_acc: 0.1247\n",
      "Epoch 9/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 1.0767 - acc: 0.1468 - val_loss: 1.7128 - val_acc: 0.1268\n",
      "Epoch 10/120\n",
      "9000/9000 [==============================] - 155s 17ms/step - loss: 1.0582 - acc: 0.1490 - val_loss: 1.7148 - val_acc: 0.1274\n",
      "Epoch 11/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 1.0402 - acc: 0.1514 - val_loss: 1.6960 - val_acc: 0.1298\n",
      "Epoch 12/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 1.0233 - acc: 0.1539 - val_loss: 1.6876 - val_acc: 0.1311\n",
      "Epoch 13/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 1.0069 - acc: 0.1561 - val_loss: 1.6788 - val_acc: 0.1325\n",
      "Epoch 14/120\n",
      "9000/9000 [==============================] - 155s 17ms/step - loss: 0.9914 - acc: 0.1574 - val_loss: 1.6795 - val_acc: 0.1345\n",
      "Epoch 15/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.9767 - acc: 0.1595 - val_loss: 1.6792 - val_acc: 0.1348\n",
      "Epoch 16/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.9622 - acc: 0.1613 - val_loss: 1.6641 - val_acc: 0.1366\n",
      "Epoch 17/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.9486 - acc: 0.1628 - val_loss: 1.6545 - val_acc: 0.1384\n",
      "Epoch 18/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.9355 - acc: 0.1645 - val_loss: 1.6520 - val_acc: 0.1376\n",
      "Epoch 19/120\n",
      "9000/9000 [==============================] - 151s 17ms/step - loss: 0.9229 - acc: 0.1664 - val_loss: 1.6516 - val_acc: 0.1378\n",
      "Epoch 20/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.9108 - acc: 0.1680 - val_loss: 1.6491 - val_acc: 0.1387\n",
      "Epoch 21/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.8991 - acc: 0.1694 - val_loss: 1.6420 - val_acc: 0.1400\n",
      "Epoch 22/120\n",
      "9000/9000 [==============================] - 151s 17ms/step - loss: 0.8873 - acc: 0.1712 - val_loss: 1.6461 - val_acc: 0.1394\n",
      "Epoch 23/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.8758 - acc: 0.1728 - val_loss: 1.6404 - val_acc: 0.1417\n",
      "Epoch 24/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.8642 - acc: 0.1739 - val_loss: 1.6435 - val_acc: 0.1417\n",
      "Epoch 25/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.8522 - acc: 0.1749 - val_loss: 1.6361 - val_acc: 0.1399\n",
      "Epoch 26/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.8406 - acc: 0.1762 - val_loss: 1.6367 - val_acc: 0.1428\n",
      "Epoch 27/120\n",
      "9000/9000 [==============================] - 155s 17ms/step - loss: 0.8292 - acc: 0.1775 - val_loss: 1.6392 - val_acc: 0.1402\n",
      "Epoch 28/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.8188 - acc: 0.1786 - val_loss: 1.6361 - val_acc: 0.1418\n",
      "Epoch 29/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.8089 - acc: 0.1795 - val_loss: 1.6336 - val_acc: 0.1421\n",
      "Epoch 30/120\n",
      "9000/9000 [==============================] - 151s 17ms/step - loss: 0.7992 - acc: 0.1807 - val_loss: 1.6339 - val_acc: 0.1429\n",
      "Epoch 31/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.7900 - acc: 0.1818 - val_loss: 1.6321 - val_acc: 0.1413\n",
      "Epoch 32/120\n",
      "9000/9000 [==============================] - 151s 17ms/step - loss: 0.7812 - acc: 0.1828 - val_loss: 1.6298 - val_acc: 0.1430\n",
      "Epoch 33/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.7725 - acc: 0.1839 - val_loss: 1.6319 - val_acc: 0.1424\n",
      "Epoch 34/120\n",
      "9000/9000 [==============================] - 158s 18ms/step - loss: 0.7642 - acc: 0.1845 - val_loss: 1.6347 - val_acc: 0.1437\n",
      "Epoch 35/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.7563 - acc: 0.1859 - val_loss: 1.6334 - val_acc: 0.1426\n",
      "Epoch 36/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.7483 - acc: 0.1864 - val_loss: 1.6311 - val_acc: 0.1428\n",
      "Epoch 37/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.7403 - acc: 0.1876 - val_loss: 1.6347 - val_acc: 0.1420\n",
      "Epoch 38/120\n",
      "9000/9000 [==============================] - 153s 17ms/step - loss: 0.7333 - acc: 0.1889 - val_loss: 1.6336 - val_acc: 0.1438\n",
      "Epoch 39/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.7263 - acc: 0.1896 - val_loss: 1.6357 - val_acc: 0.1419\n",
      "Epoch 40/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.7189 - acc: 0.1906 - val_loss: 1.6396 - val_acc: 0.1402\n",
      "Epoch 41/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.7123 - acc: 0.1915 - val_loss: 1.6335 - val_acc: 0.1430\n",
      "Epoch 42/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.7053 - acc: 0.1926 - val_loss: 1.6379 - val_acc: 0.1425\n",
      "Epoch 43/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.6993 - acc: 0.1929 - val_loss: 1.6358 - val_acc: 0.1438\n",
      "Epoch 44/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6930 - acc: 0.1941 - val_loss: 1.6382 - val_acc: 0.1431\n",
      "Epoch 45/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.6864 - acc: 0.1948 - val_loss: 1.6412 - val_acc: 0.1445\n",
      "Epoch 46/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6800 - acc: 0.1958 - val_loss: 1.6411 - val_acc: 0.1432\n",
      "Epoch 47/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6742 - acc: 0.1965 - val_loss: 1.6500 - val_acc: 0.1429\n",
      "Epoch 48/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6681 - acc: 0.1969 - val_loss: 1.6493 - val_acc: 0.1428\n",
      "Epoch 49/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.6623 - acc: 0.1981 - val_loss: 1.6589 - val_acc: 0.1395\n",
      "Epoch 50/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.6568 - acc: 0.1985 - val_loss: 1.6557 - val_acc: 0.1432\n",
      "Epoch 51/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6514 - acc: 0.1993 - val_loss: 1.6524 - val_acc: 0.1415\n",
      "Epoch 52/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.6455 - acc: 0.2004 - val_loss: 1.6577 - val_acc: 0.1408\n",
      "Epoch 53/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6405 - acc: 0.2010 - val_loss: 1.6593 - val_acc: 0.1410\n",
      "Epoch 54/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6355 - acc: 0.2015 - val_loss: 1.6617 - val_acc: 0.1409\n",
      "Epoch 55/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.6297 - acc: 0.2027 - val_loss: 1.6639 - val_acc: 0.1419\n",
      "Epoch 56/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6247 - acc: 0.2031 - val_loss: 1.6764 - val_acc: 0.1408\n",
      "Epoch 57/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.6198 - acc: 0.2040 - val_loss: 1.6695 - val_acc: 0.1403\n",
      "Epoch 58/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.6145 - acc: 0.2046 - val_loss: 1.6776 - val_acc: 0.1402\n",
      "Epoch 59/120\n",
      "9000/9000 [==============================] - 153s 17ms/step - loss: 0.6097 - acc: 0.2050 - val_loss: 1.6850 - val_acc: 0.1386\n",
      "Epoch 60/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.6046 - acc: 0.2058 - val_loss: 1.6840 - val_acc: 0.1402\n",
      "Epoch 61/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.5997 - acc: 0.2066 - val_loss: 1.6742 - val_acc: 0.1421\n",
      "Epoch 62/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.5946 - acc: 0.2073 - val_loss: 1.6827 - val_acc: 0.1400\n",
      "Epoch 63/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5898 - acc: 0.2081 - val_loss: 1.6907 - val_acc: 0.1387\n",
      "Epoch 64/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5847 - acc: 0.2090 - val_loss: 1.6919 - val_acc: 0.1395\n",
      "Epoch 65/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.5802 - acc: 0.2099 - val_loss: 1.6904 - val_acc: 0.1392\n",
      "Epoch 66/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.5755 - acc: 0.2105 - val_loss: 1.6889 - val_acc: 0.1411\n",
      "Epoch 67/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.5709 - acc: 0.2110 - val_loss: 1.6931 - val_acc: 0.1396\n",
      "Epoch 68/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.5664 - acc: 0.2122 - val_loss: 1.7031 - val_acc: 0.1375\n",
      "Epoch 69/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.5620 - acc: 0.2129 - val_loss: 1.6968 - val_acc: 0.1396\n",
      "Epoch 70/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.5580 - acc: 0.2132 - val_loss: 1.7026 - val_acc: 0.1401\n",
      "Epoch 71/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.5535 - acc: 0.2139 - val_loss: 1.7059 - val_acc: 0.1410\n",
      "Epoch 72/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5490 - acc: 0.2147 - val_loss: 1.7058 - val_acc: 0.1378\n",
      "Epoch 73/120\n",
      "9000/9000 [==============================] - 153s 17ms/step - loss: 0.5449 - acc: 0.2155 - val_loss: 1.7115 - val_acc: 0.1392\n",
      "Epoch 74/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.5405 - acc: 0.2162 - val_loss: 1.7248 - val_acc: 0.1380\n",
      "Epoch 75/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5363 - acc: 0.2167 - val_loss: 1.7093 - val_acc: 0.1385\n",
      "Epoch 76/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.5323 - acc: 0.2174 - val_loss: 1.7186 - val_acc: 0.1382\n",
      "Epoch 77/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5281 - acc: 0.2182 - val_loss: 1.7249 - val_acc: 0.1388\n",
      "Epoch 78/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.5241 - acc: 0.2188 - val_loss: 1.7220 - val_acc: 0.1381\n",
      "Epoch 79/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.5203 - acc: 0.2191 - val_loss: 1.7212 - val_acc: 0.1376\n",
      "Epoch 80/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5166 - acc: 0.2201 - val_loss: 1.7290 - val_acc: 0.1378\n",
      "Epoch 81/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5129 - acc: 0.2208 - val_loss: 1.7284 - val_acc: 0.1368\n",
      "Epoch 82/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.5092 - acc: 0.2211 - val_loss: 1.7230 - val_acc: 0.1376\n",
      "Epoch 83/120\n",
      "9000/9000 [==============================] - 155s 17ms/step - loss: 0.5057 - acc: 0.2220 - val_loss: 1.7300 - val_acc: 0.1377\n",
      "Epoch 84/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.5021 - acc: 0.2226 - val_loss: 1.7411 - val_acc: 0.1368\n",
      "Epoch 85/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4991 - acc: 0.2230 - val_loss: 1.7256 - val_acc: 0.1391\n",
      "Epoch 86/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4955 - acc: 0.2239 - val_loss: 1.7350 - val_acc: 0.1359\n",
      "Epoch 87/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4919 - acc: 0.2243 - val_loss: 1.7328 - val_acc: 0.1368\n",
      "Epoch 88/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4888 - acc: 0.2251 - val_loss: 1.7246 - val_acc: 0.1385\n",
      "Epoch 89/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.4857 - acc: 0.2251 - val_loss: 1.7303 - val_acc: 0.1368\n",
      "Epoch 90/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4825 - acc: 0.2258 - val_loss: 1.7388 - val_acc: 0.1377\n",
      "Epoch 91/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4791 - acc: 0.2269 - val_loss: 1.7286 - val_acc: 0.1404\n",
      "Epoch 92/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4761 - acc: 0.2270 - val_loss: 1.7435 - val_acc: 0.1367\n",
      "Epoch 93/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4727 - acc: 0.2277 - val_loss: 1.7277 - val_acc: 0.1401\n",
      "Epoch 94/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4702 - acc: 0.2282 - val_loss: 1.7397 - val_acc: 0.1383\n",
      "Epoch 95/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4672 - acc: 0.2289 - val_loss: 1.7424 - val_acc: 0.1363\n",
      "Epoch 96/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4644 - acc: 0.2292 - val_loss: 1.7483 - val_acc: 0.1363\n",
      "Epoch 97/120\n",
      "9000/9000 [==============================] - 150s 17ms/step - loss: 0.4616 - acc: 0.2303 - val_loss: 1.7479 - val_acc: 0.1368\n",
      "Epoch 98/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4585 - acc: 0.2305 - val_loss: 1.7519 - val_acc: 0.1378\n",
      "Epoch 99/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4559 - acc: 0.2308 - val_loss: 1.7618 - val_acc: 0.1362\n",
      "Epoch 100/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4533 - acc: 0.2316 - val_loss: 1.7501 - val_acc: 0.1378\n",
      "Epoch 101/120\n",
      "9000/9000 [==============================] - 153s 17ms/step - loss: 0.4505 - acc: 0.2319 - val_loss: 1.7594 - val_acc: 0.1379\n",
      "Epoch 102/120\n",
      "9000/9000 [==============================] - 153s 17ms/step - loss: 0.4482 - acc: 0.2320 - val_loss: 1.7588 - val_acc: 0.1360\n",
      "Epoch 103/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.4448 - acc: 0.2329 - val_loss: 1.7517 - val_acc: 0.1388\n",
      "Epoch 104/120\n",
      "9000/9000 [==============================] - 151s 17ms/step - loss: 0.4422 - acc: 0.2335 - val_loss: 1.7598 - val_acc: 0.1362\n",
      "Epoch 105/120\n",
      "9000/9000 [==============================] - 151s 17ms/step - loss: 0.4393 - acc: 0.2336 - val_loss: 1.7618 - val_acc: 0.1382\n",
      "Epoch 106/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.4371 - acc: 0.2344 - val_loss: 1.7736 - val_acc: 0.1349\n",
      "Epoch 107/120\n",
      "9000/9000 [==============================] - 158s 18ms/step - loss: 0.4345 - acc: 0.2347 - val_loss: 1.7685 - val_acc: 0.1380\n",
      "Epoch 108/120\n",
      "9000/9000 [==============================] - 156s 17ms/step - loss: 0.4319 - acc: 0.2350 - val_loss: 1.7687 - val_acc: 0.1351\n",
      "Epoch 109/120\n",
      "9000/9000 [==============================] - 156s 17ms/step - loss: 0.4293 - acc: 0.2353 - val_loss: 1.7748 - val_acc: 0.1375\n",
      "Epoch 110/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4268 - acc: 0.2360 - val_loss: 1.7755 - val_acc: 0.1359\n",
      "Epoch 111/120\n",
      "9000/9000 [==============================] - 152s 17ms/step - loss: 0.4242 - acc: 0.2367 - val_loss: 1.7707 - val_acc: 0.1373\n",
      "Epoch 112/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4215 - acc: 0.2367 - val_loss: 1.7819 - val_acc: 0.1358\n",
      "Epoch 113/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4191 - acc: 0.2374 - val_loss: 1.7795 - val_acc: 0.1355\n",
      "Epoch 114/120\n",
      "9000/9000 [==============================] - 146s 16ms/step - loss: 0.4169 - acc: 0.2380 - val_loss: 1.7833 - val_acc: 0.1369\n",
      "Epoch 115/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4141 - acc: 0.2380 - val_loss: 1.7870 - val_acc: 0.1357\n",
      "Epoch 116/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4121 - acc: 0.2389 - val_loss: 1.7867 - val_acc: 0.1355\n",
      "Epoch 117/120\n",
      "9000/9000 [==============================] - 147s 16ms/step - loss: 0.4091 - acc: 0.2393 - val_loss: 1.7787 - val_acc: 0.1368\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4068 - acc: 0.2398 - val_loss: 1.7885 - val_acc: 0.1358\n",
      "Epoch 119/120\n",
      "9000/9000 [==============================] - 149s 17ms/step - loss: 0.4047 - acc: 0.2400 - val_loss: 1.8014 - val_acc: 0.1353\n",
      "Epoch 120/120\n",
      "9000/9000 [==============================] - 148s 16ms/step - loss: 0.4024 - acc: 0.2402 - val_loss: 1.7896 - val_acc: 0.1352\n"
     ]
    }
   ],
   "source": [
    "h2 = model.fit([encoder_input_data, decoder_input_data], decoder_output_data,\n",
    "              batch_size=128, epochs=120, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x2aa78492cf8>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl0VdX5xvHvC4RJQEVAJiGgIKNCDU4oKsqk1HkJDtSpUGurlVarVmutrdWKP6u2KEWlSB1RaZ1wqqWiFZGgyKiIIBpECKjIPOX9/fEmTcBAAtzkDnk+a92V3HM29+yTG567s88+e5u7IyIimaVasisgIiKJp3AXEclACncRkQykcBcRyUAKdxGRDKRwFxHJQAp3EZEMpHAXEclACncRkQxUI1kHbtSokWdnZyfr8CIiaWn69Okr3L1xWeWSFu7Z2dnk5uYm6/AiImnJzBaXp5y6ZUREMpDCXUQkAyncRUQykMJdRCQDKdxFRDKQwl1EJAMp3EVEMlCZ4W5mY8xsuZnN3sH+vc3seTP7wMzmmNnFia9mCStWwPDhsHp1hR5GRCSdlaflPhbov5P9PwHmuvuhwPHA/5lZzT2v2g68/jrcey906wZvv11hhxERSWdlhru7Twa+2lkRoL6ZGVCvsOyWxFSvFIMGweTJ4A7HHgvXXw8bNlTY4URE9tiGDfD555V6yET0uf8F6Ah8AcwCfubuBQl43R3r2RNmzICLLoLbb4dDD4U33qjQQ4qI7Jbp06F7d8jOhmuugXXrKuWwiQj3fsAMoDnQDfiLmTUoraCZDTOzXDPLzc/P37OjNmgADz0Er74KmzfD8cdD69YweDA8//yevbaISFnc4ZNP4N134f334aOPYluRjRvh97+HI4+Ma4SDB8Odd0LXrtG9XMESMXHYxcDt7u7AAjNbBHQA3t2+oLuPBkYD5OTk+Pb7d0ufPjBrFvztb/Dmm/F46il44QUYMCAhhxCRKurDD6Ox2Lw5tG0Ly5dHmL/7brTIv/562/JdusBVV0FWFtx0EyxeHF3J990HDRvC0KEwbBi89x6ceGKFVt3cy85YM8sGXnD3LqXsux9Y5u43m9n+wHvAoe6+YmevmZOT4xUyK+TatXDMMbBwIbzzDnTsmPhjiEj6+PjjCOHDDoPq1bfdN3cuvPgi5OdHduy1V/QC9OgB99wDd9wRPQMl1agRre8ePeI1W7SALVtg6VIYNQo++CDKfe978Mc/wkknbfvv16+P8K+xe21rM5vu7jlllisr3M3scWIUTCNgGfAbIAvA3UeZWXNiRE0zwIhW/CNlHbjCwh3gs8/iB9+gQQT8fvtVzHFEJHWtXw+//W10hWzdGi3n446LAN+6FWbOhDlzomzt2rF9zZroTikyZEh0raxZEw3Ghg2j/7xOndKP6R4DPtauhf79oVribyVKWLhXlAoNd4hhkr17w8EHw2uvQZMmFXcsEUmstWth5MgI2p//PIIX4NNPo8FWq1Zs69ED9t039m3ZEq3wDz6AJUvgX/+KQL700ugCefVV+O9/o1z16nDAAXDWWXDGGdHtAvGB8PbbUe644+KRYhTuEKF+2mnQqlVcwGjRomKPJyJ7ZskSePZZ+N3v4MsvY1t2doyKmzQpBlFsKTHSunZtOPNM6NwZRo+OPm6IxtxBB8Xr9O5d6adRkcob7klbialS9OkDr7wCp5wSf0pdcAGcey7k5IBZsmsnkpkKCqIFXNTaLm3/22/D00/DW29F10XNmhHMeXlR5phjYMKE6D4ZOjRGmmRlxcXIoUOjzFdfRZlHH4XHHoNeveDuu2MgRa1alXOuKSyzW+5F3n8fbrkFJk6ETZviz7TRoyukP0wkrUybFv3SgwdH42dXFBTAyy9HAA8cGA2mL76IlvT8+fCPfxR3a7zxRvyfmz8/Ht9+GwHcs2cE+8aN0LRpDBs86qhtG2AbN8aIlR49Yrjz9jZsiFZ+FVmTubwtd9w9KY/DDjvMK91XX7lffbU7uF95pXtBQeXXQaSyLFvmvny5+7ffum/d+t19P/qRu5l7jRrxf+KGG9w3bXJ/+GH3jh3dW7d2P+4492HD3KdOLf63337r/re/RZm4hOh++OHuY8a4N2vmvtde7gcd5J6V5T56tPuPfxxlmjRx79vX/Sc/cX/ssXgd2WVArpcjY6tWuLtHoA8fHqd+3XXf/aUXSRWTJ7s/+mjpjZAFC9xHjHD/05/cX3rJ/eOPI7BXrXJ/4gn3nj2LgxfcGzVyHzLEfdQo9zPOiECvXt39qqvcV6xwv/TSKNewYXzt3t39/PPdjznGvX792Nazp/uAAe41a8bzQw6J+o0Z4968eWxr08Z95sxoSJ1wQmwzc//5z93Xrq38n2EGUrjvTEGB+9ChxS2OadNi+/r17t98k7x6ibhH6/m66yIUIVrOmza5b94creoePbYN7tIeBx7ofuut7n/5S3wIXHBBcXA3bhxhO29e8TELCtzvuisC/Jlntv1AWbXK/e67ozV+4IHROHrjjW3LrFnjPm5cfFAU2bgxjj1lSsX/zKqQ8oZ71ehzL407PPJIzPWwfHmMX125MoZIPfQQXHhh8uomVdOKFXFn9ciRkJsLP/whNGoUI0V69ozf048/jhtofvADOOecGG89bx4sWhRjsdeujbskSxtjvXVr3CLfrl1cnJS0pNEyZTGLGxROPRX+7//iDrWWLWPI5CWXxEWec89Ndi0lnW3cCE8+GQF90knxO/Xtt3ER8t13Y9qMjz+O0IUYKVJQEOOvn3wywhugQ4cYIdKpU1ykPO20bUd7NW4cI0XKUr16vIZUCVW35b4j69bBySfHEK377oug383bhKWK2roVnngCbrwxbrqBuNHmkENgypQYsVWrVozN7tAhQh8i1E87LW5b336o7jffxB3XGuFV5ekmpj2xZk2MjZ88OYZeXXVVtJx2NG5XMt/MmfDnP8ew2q+/jln+GjSIVnOjRnHTzD77wOzZEeCrV8e9FbfdFjfdPPFEvMZJJ8Hpp8dwPzUaZDco3PdUQUH0f955Z8w0ud9+sbzf4MExkdD69dG6ysqCZs2iz17Sy+LF8MADMG5c9FXvvXe0nn/5y/jrDaIL5c474d//hrp1o/ujYUOoXz+6WPLzix8rV8Z0Fz17Qt++0eWnlrYkmMI9kaZMgVtvjXkrSlOnDowdW9xHKqlj06aYmW/Bgvhg7tMnPqzvvrt43v8BA6BNG1i1Ku6cXLgwAnrVqmiJt2wJV1wRFzj1IS5JpguqiXTUUdGKnzEj5nCuWzfmtIBoxd97b8zZPHs23HyzWmupYubMuGg+c2Z0oYwbF+/dunXRlfKrX0V3W8m7HjdvjtFSf/hDdLM8/HBcWNfoEkkzarknwsaNcPnlMGZMfBD84Q8xJ7TsmS+/jA/MTp2i62v7i4z5+TEEcPnymEt70aJ4fPppdLksXQr77x9dL/36xV9ezz8frfLzztvxtK0iKUzdMpXNPbpmbrwx5tc49tiYyP/AA6O/vmbNaDW2bh1dAAqWbW3dGn3YdetG//cf/xgXMNevj/0NG8b47q5do7/71Vfjr6iSatSIGUCzs+PnfNBBMdFUo0aVfjoiFUXhnizr18dNKOPGxfqKO1oM95BD4OKL40/+pUtjjuqNG2OUzkEHRZlVq2JSpP33r7z6V7aNG2OJxNtvL56uFaKVfv758ViwILpWZs2Klvy6dTHB1IABMSKlSZN4tGihESiS8RTuqcAdli2LMcqbNsUQy8WLI6yefz5m5CtN+/YRYEXTn/btCz/6USwZuGFDtHA/+SQedepEt0XnzvEXQdGY6Yo+rxUr4i7KqVOjTqefDkccEX3W06dH3Zo3j9Ena9ZEd8knn8SyZnPnxnBCsxhhsnJl/Nuzz46f06ZNsYhC167fPXZBQRyvbt2KP0+RFKRwTwezZkXIt2kTLVGzWKjg1VejG6Jz5wiyhx4qDvqSqlcvvrsR4kLuAQdEN9CWLRGEjRpFf3XLltFd0apVfDgsXhyB+9FHcZckxDEbN44ujezsCODp02OR4Dp1Yqjg5s1Rl6K/SKpVi3ps3hzH+eab4q6U0jRtGufVuHF8SNSqFRc9TzxRc+yLlIPCPZNs3RrjrL/6qnitxzZtIqg3bozwnTu3uDX/zTfFozvy86PbJy8vWsQlNWoU47Lbt4+AXrkyLk4uXhwr4tSpA926xVwlmzfH6xYtT9aqVezLyYn6PfdcjChq1izGgnfsGBdEP/sM6tWL+rZpU7wkmojslkQukD0GGAgsd/cuOyhzPHA3sXD2Cncvc+FBhXslKygoDtuim3Xq1dtx+U2bIsi3Xy1eRJIqkePcxwJ/Acbt4ED7APcB/d39MzPTStSpqFq16AMvWgi4LJXRdy8iFabMu23cfTLw1U6KnAdMcPfPCssvT1DdRERkNyXiVsr2wL5m9h8zm25mP0jAa4qIyB5IxKDgGsBhwIlAHWCKmb3j7vO3L2hmw4BhAK1atUrAoUVEpDSJaLnnAS+7+1p3XwFMBg4traC7j3b3HHfPady4cQIOLSIipUlEuD8LHGtmNcysLnAEMC8BrysiIrupzG4ZM3scOB5oZGZ5wG+IIY+4+yh3n2dmLwMzgQLgQXefXXFVFhGRspQZ7u5e5kKi7j4CGJGQGomIyB7TxOMiIhlI4S4ikoEU7iIiGUjhLiKSgRTuIiIZSOEuIpKBFO4iIhlI4S4ikoEU7iIiGUjhLiKSgRTuIiIZSOEuIpKBFO4iIhlI4S4ikoEU7iIiGUjhLiKSgRTuIiIZSOEuIpKBFO4iIhmozHA3szFmttzMdrrotZn1MLOtZnZ24qonIiK7ozwt97FA/50VMLPqwB+BVxJQJxER2UNlhru7Twa+KqPYFcAzwPJEVEpERPbMHve5m1kL4Axg1J5XR0REEiERF1TvBq51961lFTSzYWaWa2a5+fn5CTi0iIiUpkYCXiMHeMLMABoBJ5vZFnf/5/YF3X00MBogJyfHE3BsEREpxR6Hu7u3KfrezMYCL5QW7CIiUnnKDHczexw4HmhkZnnAb4AsAHdXP7uISAoqM9zd/dzyvpi7X7RHtRERkYTQHaoiIhlI4S4ikoEU7iIiGUjhLiKSgRTuIiIZSOEuIpKBFO4iIhlI4S4ikoEU7iIiGUjhLiKSgRTuIiIZSOEuIpKBFO4iIhkoLcN95cpk10BEJLWlXbiPHw/Z2TBnTrJrIiKSutIu3I87DurUgfPOg40bk10bEZHUlHbhvv/+MGYMzJwJv/pVsmsjIpKa0i7cAQYOhMsvh7vugn/9K9m1ERFJPWkZ7gAjRkDHjjBkCCxbluzaiIikljLD3czGmNlyM5u9g/3nm9nMwsfbZnZo4qv5XXXrwpNPwjffwAUXwNatlXFUEZH0UJ6W+1ig/072LwKOc/dDgN8BoxNQr3Lp2hX+/Ofomrnttso6qohI6isz3N19MvDVTva/7e5fFz59B2iZoLqVy6WXxsiZ3/wG/vOfyjyyiEjqSnSf+6XASwl+zZ0yg1GjoF07GDwYli6tzKOLiKSmhIW7mZ1AhPu1OykzzMxyzSw3Pz8/UYemfn145hlYvToCfsuWhL20iEhaSki4m9khwIPAae6+w8kB3H20u+e4e07jxo0Tcej/6dw5WvCTJ8MNNyT0pUVE0s4eh7uZtQImAEPcff6eV2n3DRkCl10Gd9wRLXkRkaqqRlkFzOxx4HigkZnlAb8BsgDcfRRwE7AfcJ+ZAWxx95yKqnBZ7r4bZsyAiy6KcfCdOiWrJiIiyWPunpQD5+TkeG5uboW89pIlcNhh0KABTJsGe+9dIYcREal0Zja9PA3otL1DdWdatICnnoJFi2DQIF1gFZGqJyPDHeDYY+G+++CVV+Dqq5NdGxGRylVmn3s6GzoU5s6NfviOHeFHP0p2jUREKkdGhzvAnXfC/Pnwk59Aq1YwYECyayQiUvEytlumSPXq8MQTcMghcPbZcYFVRCTTZXy4Q9zBOnEiNGkCp5wCCxYku0YiIhWrSoQ7QNOmcXG1oAD69dMcNCKS2apMuAO0bx8t+GXLoH//mAteRCQTValwBzj8cPjHP2DePPj+92HdumTXSEQk8apcuAP06QOPPAL//S+cfjps2JDsGomIJFaVDHeAc86BMWPgtddiFM2mTcmukYhI4lTZcIeYXGzUKHjxxZgHfvPmZNdIRCQxqnS4Q9y1eu+90Q9/7rkKeBHJDFU+3AGuuAL+9KeYA/788zXRmIikv4yffqC8rroqxsD/4hfgDo89BllZya6ViMjuUbiX8POfx4LbP/85bN0a0xbUrJnsWomI7Dp1y2xn+HC4557ogz/rLFi/Ptk1EhHZdQr3Ulx5Jdx/f4yi6ddPd7KKSPpRuO/AZZfB44/DO+/A8cdrLhoRSS9lhruZjTGz5WY2ewf7zczuNbMFZjbTzL6X+Gomx6BB8MILMYvk0UfHvPAiIumgPC33sUD/newfALQrfAwD7t/zaqWOvn1h0iRYuxZ69oSpU5NdIxGRspUZ7u4+GfhqJ0VOA8Z5eAfYx8yaJaqCqaBHD3j7bWjQAE44IcbDi4ikskT0ubcAPi/xPK9w23eY2TAzyzWz3Pz8/AQcuvIcdBBMmQLdusVcNLfdFuPhRURSUSLC3UrZVmrsuftod89x95zGjRsn4NCVq0kT+Pe/Y5qCX/0KhgzRUEkRSU2JCPc84IASz1sCXyTgdVNS7drw6KNwyy1xF2vPnrB4cbJrJSKyrUSE+3PADwpHzRwJrHL3jB44aAa//jU89xx88gnk5ESLXkQkVZRnKOTjwBTgYDPLM7NLzewyM7ussMhEYCGwAHgAuLzCaptiBg6EadOiu6ZPH7jzTvXDi0hqKHNuGXc/t4z9DvwkYTVKM+3bx41Ol1wC11wTF10fegj22SfZNRORqkx3qCZA/fowfny03J97Drp313h4EUkuhXuCmMV0wW+9FV0zxxwDt98e0wiLiFQ2hXuCHXEEvP8+nHEGXH999MUvWZLsWolIVaNwrwD77gtPPhkLcE+dCl26xCRkIiKVReFeQczg4oujFd+hA5x3XkxEtmJFsmsmIlWBwr2CtWsHb74Jt94aC4B06gRPPZXsWolIplO4V4IaNWK6gunToVUrOOecWOVJc8SLSEVRuFeirl1jTPztt8PEidCxIzzwgEbUiEjiKdwrWY0acO21MHNmjIcfNgxOPDEWBBERSRSFe5K0axfz0TzwQFx07doVRoyAzZuTXTMRyQQK9yQygx/+EObOjYW4f/nLmITsnXeSXTMRSXcK9xTQvHmMpJkwAVaujPVaf/pT+PbbZNdMRNKVwj1FmMVdrfPmwRVXwH33xbDJZ59Nds1EJB0p3FNM/fpwzz3RNbPffnD66TB4MKTZqoQikmQK9xR1+OGQmwu/+11013TqFCs/ab54ESkPhXsKy8qCG2+E996DNm3g/PPh5JPh00+TXTMRSXUK9zTQpUssAnLPPTGVQefOGjYpIjuncE8T1avDlVfGsMmTTioeNvn228mumYikIoV7mmnVKkbQ/OMf8NVX0LMnnHsuLF6c7JqJSCopV7ibWX8z+8jMFpjZdaXsb2Vmk8zsfTObaWYnJ76qUtLpp8OHH8JNN8E//wkHHxzTGnz9dbJrJiKpoMxwN7PqwEhgANAJONfMOm1X7EZgvLt3BwYD9yW6ovJde+0Fv/0tzJ8fM02OGAFt28Idd8C6dcmunYgkU3la7ocDC9x9obtvAp4ATtuujAMNCr/fG/gicVWUshxwAIwbBzNmxN2t114bc9f89a+66CpSVZUn3FsAn5d4nle4raSbgQvMLA+YCFxR2guZ2TAzyzWz3HzdlZNwhxwCL74Ib7wB2dlw2WXQvj089JBCXqSqKU+4Wynbtr+V5lxgrLu3BE4G/m5m33ltdx/t7jnuntO4ceNdr62US69e8NZb8MIL0LhxTE7Wrh2MGgUbNiS7diJSGcoT7nnAASWet+S73S6XAuMB3H0KUBtolIgKyu4xg1NOiQW6X3gBmjaFH/8YDjwQ7roL1qxJdg1FpCKVJ9ynAe3MrI2Z1SQumD63XZnPgBMBzKwjEe7qd0kBRSE/ZQr8618xquYXv4DWreNirBbsFslMZYa7u28Bfgq8AswjRsXMMbNbzOzUwmK/AIaa2QfA48BF7poFJZWYxYpP//53BP0xx8DNN8e4+csvjxE3IpI5LFkZnJOT47m5uUk5toR586KLZtw42LQJBgyIu2D79oVqur1NJCWZ2XR3zymrnP4LV2FFC3R/9ll00bz/fgR8p05w//2wdm2yaygiu0vhLuy/f9zpungx/P3vMaf85ZdDy5YxZv7zz8t+DRFJLQp3+Z+aNeGCC+Ddd+G//40Jyu68M6Yb/v73Y175TZuSXUsRKQ+Fu3yHWdzp+tRT8MkncPXVMH06nHUWtGgBw4fD7NnJrqWI7IzCXXYqOxtuvz365V98EY4/HkaOhK5d4aijYOxYzWMjkooU7lIuNWrEKlBPPQVffBGjbL75Bi6+GJo1g0svhUmToKAg2TUVEVC4y25o1Ci6ZubOhf/8B848M0K/d++YlfLmm2HRomTXUqRqU7jLbjOD446Dv/0NvvwSHn887oC95ZYI+cMPj+mHFfQilU/hLglRty4MHgyvvBILeN92G7jHUMq2beHII2MN2GXLkl1TkapB4S4J16oVXHcdTJsWrfbbb4eNG+Gqq2K0zYAB8MgjsHp1smsqkrkU7lKhsrOj9f7++zBnTnw/dy4MGRI3Tw0aBOPHK+hFEk3hLpWmUye49dZozb/5Zoy0mTQpAr5x47hRaty4GIUjIntG4S6Vrlq1mJVy5EhYujRWjrrsMvjgA7jwwmjRn3wyPPggLF+e7NqKpCfNCikpo6Agpj54+umY6mDRohiRc9RR0aofOBA6d45tIlVVeWeFVLhLSnKPlvyzz8Lzz8f0BxCLjJxySoyt79ULsrKSW0+RyqZwl4yyZAlMnBhTILz2Wkx50LBhjLzp2zceTZsmu5YiFU/hLhlr3boYTz9hQnzNL1zQsUsX6NMnAv+442KWS5FMo3CXKqGgILpvXnstHm++GWPq9947+ugHDIjAb9Ik2TUVSQyFu1RJ69bFQuATJsALL8DKlbG9e/cI+T594NhjoVat5NZTZHcldJk9M+tvZh+Z2QIzu24HZc4xs7lmNsfMHtvVCoskQt26cOqpMRXxsmUx+uZ3v4vVpf70pwj3hg2jzP33w4IFcfFWJNOU2XI3s+rAfKAPkAdMA85197klyrQDxgO93f1rM2vi7jsdoayWu1S2NWtiFsuXX44Ls59+GttbtYoLsv36wYknwr77JrOWIjuXsG4ZMzsKuNnd+xU+vx7A3W8rUeYOYL67P1jeCpYW7ps3byYvL48NGzaU92WqpNq1a9OyZUuyNA5wt7nDxx/D669HN87rr8OqVXGD1WGHxRKDffpAz566MCupJZHhfjbQ391/WPh8CHCEu/+0RJl/Eq37nkB14sPg5VJeaxgwDKBVq1aHLV68eJv9ixYton79+uy3336Y7lQplbuzcuVKVq9eTZs2bZJdnYyxZQtMnQqvvhpB/847sHUr1KsHJ5wQYd+7t26ikuQrb7jXKM9rlbJt+0+EGkA74HigJfCmmXVx921mCXH30cBoiJb79i+6YcMGsrOzFew7YWbst99+5BeN/5OEqFEjWuk9e8JvfxsTmU2aFF04r74aN1JBjKXv1w/6948lBzW2XlJVecI9DzigxPOWwBellHnH3TcDi8zsIyLsp+1qhRTsZdPPqOLVrx8XXU89NZ4vXhwt+qKgf/jh2H7wwTGmvnfvCPv9909alUW2UZ7RMtOAdmbWxsxqAoOB57Yr80/gBAAzawS0BxYmsqKVpV69esmugqSg1q3hkkvgiSdiMrOpU2OVqYMOim2DB0crvnNn+OlP4Zlnim+uEkmGMlvu7r7FzH4KvEL0p49x9zlmdguQ6+7PFe7ra2Zzga3ANe6+siIrLpIs1avHEoKHHw7XXBP99e+9F904kybFsoMjR0bZTp1iDpxjj42ZMA84QH32UjlS6iamefPm0bFjx6TUp0i9evVYs2YN7s4vf/lLXnrpJcyMG2+8kUGDBrF06VIGDRrEt99+y5YtW7j//vs5+uijufTSS8nNzcXMuOSSSxg+fHiF1jMVflZSuk2bIDcXJk+O6Yz/+9/ixUiaN49ZLnv2jO6cQw+NDwuR8krkBdWkuOoqmDEjsa/ZrRvcfXf5yk6YMIEZM2bwwQcfsGLFCnr06EGvXr147LHH6NevHzfccANbt25l3bp1zJgxgyVLljB79mwAvtFqE1VazZpw9NHxuO66GHUzcya89VaMwpkyJbptABo0iBZ9r17x9bDDoHbt5NZfMkPKhnuyvfXWW5x77rlUr16d/fffn+OOO45p06bRo0cPLrnkEjZv3szpp59Ot27daNu2LQsXLuSKK67glFNOoW/fvsmuvqSQ6tVj+oPu3eGKK2LbkiXRqn/jjZgPZ+LE2J6VBd/7XnTjFHXlNGyYvLpL+krZcC9vC7ui7Ki7qlevXkyePJkXX3yRIUOGcM011/CDH/yADz74gFdeeYWRI0cyfvx4xowZU8k1lnTSogWcd148IC7STpkCb78dj3vvhTvvjH2dOkU3ztFHw5FHQvv2cbOVyM6kbLgnW69evfjrX//KhRdeyFdffcXkyZMZMWIEixcvpkWLFgwdOpS1a9fy3nvvcfLJJ1OzZk3OOussDjzwQC666KJkV1/STJMmcNpp8QDYsCHmxXnrrXiMHw8PPBD79tkn+u2PPjq+Hn54DN0UKUnhvgNnnHEGU6ZM4dBDD8XMuOOOO2jatCkPP/wwI0aMICsri3r16jFu3DiWLFnCxRdfTEFBAQC33XZbGa8usnO1a0c/fK9e8bygAD76KPrs3347WvkvvRT7qlWLIZg9ekBOTjwOOUQzX1Z1Gi2TpvSzkm++ifH2U6ZE6E+fDitWxL6sLOjatTjsc3LiA0Dz5KS/tB8tIyI7t88+MRVCv37x3B0++yyGYebmwrRp0Z0zenTsr1kzWvRFF3e/9714XqdO8s5BKo7CXSRDmMWdtK1bw1lnxTaoyk/kAAALn0lEQVR3+OSTaNVPnx6h/9RTxf331avH8oRdu8aF286dI/RbtNDNVulO4S6SwcxiioSDDoJBg2JbUQu/KPCnT48hmY88UvzvGjeOkC9q5XfpAu3aRXePpAeFu0gVU7KFf+aZxdu//RZmz46pFN57D95/P4ZjbtkS+7OyYqK0zp2LW/tdukCbNhqamYoU7iICxN2yRXfWFtm4EebNgzlzYNas+Dp1Kjz5ZHGZevWi7/7QQyP4O3aMr5ohM7kU7iKyQ7VqxbQd3bptu331apg7NwJ/5syYKuSxx2I1qyJNm8a/69ABDjwwunU6dYKWLdWfXxkU7iKyy+rXhyOOiEcRd1i6tDj0P/ggHpMnw7p1xeXq1YvAP/jg+Nq+ffGjbt3KP5dMpXDfA0UzSJbm008/ZeDAgf+bTEwk05nFrJfNm8eyhEXcYdkymD+/uIvnww9jTp1HH93237dpE106Bx8cLf127eJicIsW6tffVQp3EalQZtFF07Rp8R23RdauhQUL4u7bouCfMydWvNq4sbhc7doR8gcfHC387Oz4IOjUKT5M1M3zXakb7kmY8/faa6+ldevWXH755QDcfPPNmBmTJ0/m66+/ZvPmzfz+97/ntKIJQMppw4YN/PjHPyY3N5caNWpw1113ccIJJzBnzhwuvvhiNm3aREFBAc888wzNmzfnnHPOIS8vj61bt/LrX/+aQUVj2EQyzF57xYXYQw/ddntBAXz+eQT/ggXw8cfR8p81C559tngED8B++8UF3aKWfvv20d3Ttm2sjVtVVeFT/67Bgwdz1VVX/S/cx48fz8svv8zw4cNp0KABK1as4Mgjj+TUU0/dpXVMRxYuyzNr1iw+/PBD+vbty/z58xk1ahQ/+9nPOP/889m0aRNbt25l4sSJNG/enBdffBGAVSWvUIlUEdWqFQ/XPPHEbfdt3RpTJi9cGEM3Z8yIrxMmFE+/ABHsLVvGa7RtWzzev+hDINNX1EzdcE/CnL/du3dn+fLlfPHFF+Tn57PvvvvSrFkzhg8fzuTJk6lWrRpLlixh2bJlNN2FZe/feustriicyLtDhw60bt2a+fPnc9RRR3HrrbeSl5fHmWeeSbt27ejatStXX3011157LQMHDuTYY4+tqNMVSUvVq0OrVvE4/vht9339dbTwP/wwunoWL47Hyy/Hxd6SmjaNUTxFj4MOKh7Vkwlz6KduuCfJ2WefzdNPP82XX37J4MGDefTRR8nPz2f69OlkZWWRnZ3Nhg0bduk1dzQ523nnnccRRxzBiy++SL9+/XjwwQfp3bs306dPZ+LEiVx//fX07duXm266KRGnJpLx9t33u6N4iqxZs203T9H3r78O48ZtW7Zhw+IunnbtouVf1M/ftGl6XNwtV7ibWX/gHmKB7Afd/fYdlDsbeAro4e65pZVJdYMHD2bo0KGsWLGCN954g/Hjx9OkSROysrKYNGkSixcv3uXX7NWrF48++ii9e/dm/vz5fPbZZxx88MEsXLiQtm3bcuWVV7Jw4UJmzpxJhw4daNiwIRdccAH16tVj7NixiT9JkSqoXr3Sx+wDrF8PixYVB/78+fEBMGkS/P3v25atWTP+amjfPi7wtm0bo3latoxte+9dOedTljLD3cyqAyOBPkAeMM3MnnP3uduVqw9cCUytiIpWls6dO7N69WpatGhBs2bNOP/88/n+979PTk4O3bp1o0OHDrv8mpdffjmXXXYZXbt2pUaNGowdO5ZatWrx5JNP8sgjj5CVlUXTpk256aabmDZtGtdccw3VqlUjKyuL+++/vwLOUkRKqlMnRt506vTdfevWRdfOokXx9dNP4/v58yP816/ftnzLlhH6bdpEa7+oC6l169hXWRd5y5zP3cyOAm52936Fz68HcPfbtit3N/Av4Grg6rJa7prPfc/oZyWSfAUFkJ8fF3g/+yz6+ufMiVb/p5/G+P6SqlePgL/iCvjFL3bvmImcz70F8HmJ53nANj1aZtYdOMDdXzCzq3dSqWHAMIBWrVqV49AiIqmrWrWYQ2f//WMWze2tWwd5eRH8RRd3Fy2CZs0qvm7lCffSxvz9r7lvZtWAPwEXlfVC7j4aGA3Rci9fFVPbrFmzGDJkyDbbatWqxdSpad07JSIJULdu8dQKla084Z4HHFDieUvgixLP6wNdgP8Ujv1uCjxnZqem60XVXdG1a1dmJPpmKxGRPVSeAT3TgHZm1sbMagKDgeeKdrr7Kndv5O7Z7p4NvAPsdrAna03XdKKfkYiUpcxwd/ctwE+BV4B5wHh3n2Nmt5jZqYmsTO3atVm5cqXCayfcnZUrV1K7du1kV0VEUliZo2UqSmmjZTZv3kxeXt4u3yRU1dSuXZuWLVuSpTXPRKqcRI6WqTRZWVm0adMm2dUQEUl7aXATrYiI7CqFu4hIBlK4i4hkoKRdUDWzfGDXZ+EKjYAVZZZKH5l0PjqX1KRzSU27cy6t3b1xWYWSFu57wsxyy3O1OF1k0vnoXFKTziU1VeS5qFtGRCQDKdxFRDJQuob76GRXIMEy6Xx0LqlJ55KaKuxc0rLPXUREdi5dW+4iIrITaRfuZtbfzD4yswVmdl2y67MrzOwAM5tkZvPMbI6Z/axwe0Mze83MPi78um+y61peZlbdzN43sxcKn7cxs6mF5/Jk4UyiKc/M9jGzp83sw8L356h0fV/MbHjh79dsM3vczGqn0/tiZmPMbLmZzS6xrdT3wsK9hXkw08xKWTIjeXZwLiMKf89mmtk/zGyfEvuuLzyXj8ys354cO63CvcR6rgOATsC5ZlbKqocpawvwC3fvCBwJ/KSw/tcBr7t7O+D1wufp4mfEbKFF/gj8qfBcvgYuTUqtdt09wMvu3gE4lDintHtfzKwFsZZxjrt3IRa1H0x6vS9jgf7bbdvRezEAaFf4GAak2qLDY/nuubwGdHH3Q4D5wPUAhVkwGOhc+G/uK8y83ZJW4Q4cDixw94Xuvgl4AjgtyXUqN3df6u7vFX6/mgiQFsQ5PFxY7GHg9OTUcNeYWUvgFODBwucG9AaeLiySFudiZg2AXsBDAO6+yd2/IU3fF2JCwDpmVgOoCywljd4Xd58MfLXd5h29F6cB4zy8A+xjZpWwiF35lHYu7v5q4VTqEOtftCz8/jTgCXff6O6LgAVE5u2WdAv30tZzbZGkuuwRM8sGugNTgf3dfSnEBwDQJHk12yV3A78ECgqf7wd8U+IXN13en7ZAPvC3wi6mB81sL9LwfXH3JcCdwGdEqK8CppOe70tJO3ov0j0TLgFeKvw+oeeSbuG+0/Vc04WZ1QOeAa5y92+TXZ/dYWYDgeXuPr3k5lKKpsP7UwP4HnC/u3cH1pIGXTClKeyLPg1oAzQH9iK6LraXDu9LeaTr7xxmdgPRVfto0aZSiu32uaRbuJe1nmvKM7MsItgfdfcJhZuXFf0pWfh1ebLqtwt6Aqea2adE91hvoiW/T2F3AKTP+5MH5Ll70armTxNhn47vy0nAInfPd/fNwATgaNLzfSlpR+9FWmaCmV0IDATO9+Lx6Ak9l3QL952u55rqCvukHwLmuftdJXY9B1xY+P2FwLOVXbdd5e7Xu3vLwnVzBwP/dvfzgUnA2YXF0uVcvgQ+N7ODCzedCMwlDd8XojvmSDOrW/j7VnQuafe+bGdH78VzwA8KR80cCawq6r5JVWbWH7iWWGt6XYldzwGDzayWmbUhLhK/u9sHcve0egAnE1eYPwFuSHZ9drHuxxB/Zs0EZhQ+Tib6ql8HPi782jDZdd3F8zoeeKHw+7aFv5ALgKeAWsmuXznPoRuQW/je/BPYN13fF+C3wIfAbODvQK10el+Ax4nrBZuJ1uylO3oviK6MkYV5MIsYJZT0cyjjXBYQfetFGTCqRPkbCs/lI2DAnhxbd6iKiGSgdOuWERGRclC4i4hkIIW7iEgGUriLiGQghbuISAZSuIuIZCCFu4hIBlK4i4hkoP8HsfZyMW3yBfgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(h2.history['loss'], 'b-')\n",
    "plt.plot(h2.history['val_loss'], 'r-')\n",
    "#plt.ylim(0,3)\n",
    "plt.legend(['loss', 'val_loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\engine\\network.py:877: UserWarning: Layer lstm_6 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_5/while/Exit_3:0' shape=(?, 50) dtype=float32>, <tf.Tensor 'lstm_5/while/Exit_4:0' shape=(?, 50) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n"
     ]
    }
   ],
   "source": [
    "model.save('eng_fra_word_model.h5')\n",
    "model.save_weights('eng_fra_word_weights.h5')\n",
    "np.savez('eng_fra_word_history.npz', [h.history, h2.history])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 번역하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'eng_words' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m----------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-39321b9c2854>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'I love you'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0ml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0ml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mword\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ml\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[1;32min\u001b[0m \u001b[0meng_words\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0meng_sen_max\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m### 영어 입력\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-3-39321b9c2854>\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'I love you'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0ml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0ml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mword\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ml\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[1;32min\u001b[0m \u001b[0meng_words\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0meng_sen_max\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m### 영어 입력\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'eng_words' is not defined"
     ]
    }
   ],
   "source": [
    "s = 'I love you'\n",
    "l = s.lower().split()\n",
    "l = [word for word in l if word in eng_words][:eng_sen_max]\n",
    "\n",
    "### 영어 입력\n",
    "eng_text = np.zeros([1,eng_sen_max])\n",
    "for t,word in enumerate(l):\n",
    "    eng_text[0,t] = eng_w2i[word]\n",
    "\n",
    "### 불어 입력\n",
    "fra_text = np.zeros([1,fra_sen_max])\n",
    "fra_text[0,0] = fra_w2i['START_']\n",
    "result = []\n",
    "\n",
    "for i in range(fra_sen_max):\n",
    "    pred_y = model.predict([eng_text,fra_text])[0,i]\n",
    "    \n",
    "    idx = np.argmax(pred_y)\n",
    "    word = fra_i2w[idx]\n",
    "    result.append(word)\n",
    "    \n",
    "    if word=='_END' or i==(fra_sen_max-1): break\n",
    "        \n",
    "    fra_text[0,i+1] = idx\n",
    "    \n",
    "display(s,l,' '.join(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}