tensorflow/practice/convnets/ConvnetsFashionMNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "\n",
    "import tensorflow.keras as keras\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Input, Activation, add, Add, Dropout, BatchNormalization\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "from tensorflow.keras.datasets import fashion_mnist\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from IPython.display import SVG\n",
    "from tensorflow.python.keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "random_state = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp0AAABaCAYAAAD6p+0bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXuwVWX9xp+jlV2sSEJSUbmoKEqakGIpXkilVAKRMNKcYXRSnFBD06wcGystm0rNqbQmNVIaR9QUjMywHFMQhcTAG17BFMVLN8s0fn/85nnXs/b+nrUvZ69z1rHn8w+bd5+99lrvft93rff7fC9dGzduhDHGGGOMMWWySV+fgDHGGGOMefPjh05jjDHGGFM6fug0xhhjjDGl44dOY4wxxhhTOn7oNMYYY4wxpeOHTmOMMcYYUzp+6DTGGGOMMaXjh05jjDHGGFM6fug0xhhjjDGl85ZW/rirq8vli+p5YePGjYOA3umft7/97QCA7bbbLrW9+OKLAIB//vOfAACtMsXX73jHO1Lb+973PgDAv/71r9T23HPPAQDeeOONTp9y6f3zlrdkw3jgwIEAgA0bNqS2119/vaXjsa/Y1wDw8ssvA8j3bYdI/QN0vo/e9ra3AQDe/e53p7YBAwYAyPcL+4tjSK+d4+U973lPavvvf/+b+xwAvPDCC508daVX51gneOtb3woA+M9//tMbX1fqGCI6zzieBg1KX5vGE9cVnSubbropAGDzzTdPbX//+98BAOvWrUttJVbI63djqJfpk/7R8fDvf/8bQPGc4XoGAO9617sAAC+99FJJZ5ejV+ZYPyfXR93R0kOnCXmy3Q92dXWl180utkOHDgUA/OAHP0ht1157LQBg+fLlAIDXXnstvccJvNtuu6W2KVOmAADWrFmT2i688EIA2cNVB2m7f5pliy22SK+PO+44AMBVV12V2p599tmWjjdy5EgAwM4775zarrvuOgClPESU2j9bb701AOCAAw5IbZ/85CcB5B8Y586dCwC47777AOSvferUqQCACRMmpDY+nPJzAHDZZZd18tSV0sdQp+HD2DPPPNMbX9dW/3D9aXbt0Xl20EEHAQCOP/741Ma1Y/Xq1QDy6xA3Oh/5yEdS29133w0AOPvss1Pbq6++2vB8Wzlnod+NoV6mI/cx0uzvM2bMmPSa96O1a9d2+/dczwDgwx/+MIDs/lcyHj+NaaqPulqZvH66D7l348aNY4Hi/ml2wdxjjz3S66OPPhpAdtMHMkskd3lAZpmjla8RDz/8MIDMWgVkD1q0eALAokWLAADf+c53UtsDDzzQ1HcITfVPO3CXzH4CgFNOOQVA/oZHCxzb9D1abDbbbLPUNmTIEADAjTfemNruuusuAKUscKl/gJ710cc//nEAwGmnnZbaeBNXCwEtUWr95KZk8ODBAIAnnngivUcL1l/+8pfU9sorrwDI99s222wDALjttttS2+zZs9u9HKW0McRzpTUXyB7GTzjhhNSm/VELb4SLFy9ObZyTTz6ZrcMTJ04EAPzjH//o4VnX0fQYarQOvf/97weQzSMA+NjHPgYg/1vzGrSNGxUdV4SbNX2g4HhSFYaqzR/+8IfUdskllwDosUWrtDH0JqHt/tlkk8xLT+8phOvpzJkzU9ucOXMA5NWTVuG9UBWbM888EwBw0UUXFX6W5xydbzd0bJ1+E5Pro+6wT6cxxhhjjCkdP3QaY4wxxpjSsbzec9qWJVRaoA/iBz/4wdRGCeBvf/tbaqM0qr6FlBkYvPDe9743vUcZTGWEot9cA0goe6k0e8cddwAAjj322IIry1G6rDVt2rT0mnLyl7/85dRG+ZPSsUqClOwY1AAAt956KwDgmmuuSW2U8m+44YaOnjt6KNuMGDEivT733HMB5F0k3vnOdwKIJTCVpbbddtvccXW88DUldf2sjkNKo5TZgczP7/TTT2/6mgJKG0O33347gHw/cnyo7Ms5SN/eY445Jr3HIBkNzON1q4/i7rvv3slTV3okr+u133TTTQDyY6hozWHwB5D9/pwrGpTIv9O1hH6vGqDE9/Xv6D/8ox/9KLVdf/313V1id1heL6bl/imSqOkbDgA77rgjgPy9hb+puprwfa7JGl+w1VZbAcjWMz2GzlOOPY5FAPjtb38LAPjMZz7T7TV0dx1CJeR1zt+o76P7ek/8bel//cc//jG10Q2PLno1x7O8bowxxhhjqkG/iF4virKk0/q+++6b2m655ZZuj0GrBNB8Kp2e7BaKmD9/fnq9/fbbAwDWr1+f2riLUUsAz1nPie+zTVPX6PUS3d3VopaZKPXJ+PHjAeSjmx988MFuj9cbqFWEu2ON7mcwC60taunk3997772p7Wc/+xkAYNiwYant+eef7/RpdwQ65APxOfK3VisDx5CO/8cffxxAZs3Uv+c41H4jas3iONTgGQYoHXbYYaltwYIFxRfVizBoSH9rtmm09gc+8AEAwOc//3kAeasl1QkNdGFfaIaAKhCtW+eff356zUwPaimigqKfjdYhWpk4z9Tyy7GjAZC0nOo45Gd0jeL8Pvnkk1Mb1QhVKEz56O8dWQYZcDl69OjUxjGl6wfHkq7dXEs41zRSnVZNDQKlhVPvWXzNMQsAM2bMAJAfe5MnT667hlazOVSBRufa7LUwu4n+brRQf/Ob30xt7KNDDjkktana0Qy2dBpjjDHGmNLxQ6cxxhhjjCmdfiGvU2qh+X2HHXZI7zE5sZrY6Zys8s7SpUsBxJK6Sgb8Lm2LPkPZup0KPkyIS0kdyCRxldL5HSp1MkhDHap5zpSr9Bg8P70eSg96XQyU0Dx60XXzeJoUuodBIj1GJTbmGVSJ9wtf+AKALF+cVlGhrKwyKI+h/Ri5WFSBK664Ir1mfk6V2RkQorkTowT3lK147cpf//pXAMWJu/UYGsj29NNPA6iWpK489thjAIBx48alNo57lY1qf3/N27nffvsByFfWofSn87RqMDiDciaQuVeo7Mn+0GuhVBkFYnCN0LWRa5hKnFGeRbbpnOY6rp894ogjAOSD/Uz5RHIti40AwN577w0gfx/h3FHJm2Mlqp7He1Gj+zLHigYSRUGSTz31FIC8JMycxuqKVwVZvVEeXbYVPXd89rOfTa9ZgIFrFJC5m2nhCroIPfLII6mNwWCnnnpqaluxYkUTV1GMLZ3GGGOMMaZ0+oWls9aqyBJsQFYtQ3dWdFjWnfnBBx8MAPjJT36S2mgF0h1FtIOgg7w6HdOxuR0OPPDA3Hnqa/0OXrdaXFhxQXcpvHY6XmvlGO4Q1QGb36V1b/fcc08AWaAEEFtfeX5HHXVUautrS2dkkY0sdrweLYvJMaJpfjgGol141aAFH8ic+CdNmpTalixZAiD/G/Ka1brL8cE+UpWAf6/HoPVTrca1fw8AZ511VkvX09usWrUKQBxwp+lc2D+a0ozQAhwF97GfqgirMKmlk2M/qnGt8yxar3j9kSrA/o0CUSLrlY4rjkk9J67ntnT2DkXKngbE8rdSZYXBmqqwcH7ousrv4D2r2SAZPSe2RcqepnxbuHAhgMzaD2T3hShwt6poQC/PW0sejx37/xmMtOIa1TGt+kWrppYlZZlRfXagyvzoo4+2fc62dBpjjDHGmNLxQ6cxxhhjjCmdfiGvq3kXyMy+ADB06FAAeXmM5vlFixaltg996EMAgG9/+9upbdmyZQCAlStXprbVq1cDAPbaa6+679PM/JQy1WTfLJSm1XQfyRd0vtfvuPzyywHknaIpjTO/5Oc+97n03gMPPAAgn3OQ36VVR773ve8BAGbNmpXaaK6PKkmoWX+nnXYCkK9S0JtoMEMkufB6BwwY0NTxonxtKrlUlYsvvhgAcMopp6Q2OtFrcBFlY3UR0apXQH4+8e+1DyhZ6ecYQKTO+VWWl4Es+EelP44nDXygywplKL1uHkP7jGOonfWht6CrgJ43pXadU3ytLhd071mzZk1qY3BVFMjJNu1nyuXqsnD44YfXfZbzVt2BNKjIlE8kq994440A8pWDGACmQbJ8X10xItm6KH90EZEbVLT+q7sMXWJUip43b17dZ3ubRi4FdF1itSB1FeNa+9Of/jS1MbhU3fF4r99yyy3rvvehhx5KbZTa6coCZPPS8roxxhhjjKk0lTXfRKkD+MRN51ggszjozpeWN/4LAPfccw+A/BM6d8777LNPajvyyCMB5Hfk/KymCWJwz+LFi1u8sqyaCdPJANkuL6r6ojXaya9//ev0mju4UaNGAcgH9rBGMVOMAJnFSuvjclejO1D2qe78uFulBQ3I+q+vLJ1qAWH/qaWEO12ee2SRUvhb6M5brb1VInJ61+pc3/jGN+o+Qwun/ta11T30uHytAW2RVYJtrN/dH6AFQOc7x4RaZjieGHikVlBet1o1OQ6rmmoLyCw7d9xxR2pjfWpWkgKyiiSNKo/RCsOxpKlsuJboPOK6pcFAX/rSlwBkay4ADB48GEDeMj98+PDCczHlo/dNQut1FBym8J7ebL3wIiJLZ/T9Omc5DvVZgvOhL4NG9d4UpZXivY7rkc5TWm1V6Zw4cSKAvOpLtPohUesnq5JpkO3MmTMBAHfeeWdqo5raLLZ0GmOMMcaY0vFDpzHGGGOMKZ1KyOvNmtPPO+88APncWkRzA1I21AAkSo5qTqf5WmVmyu8qPZ588skA8pKO5qlsBjWDM6gjCiTSvqA8pfkUo+NR9mS/qKTK40XyYSSPqMMxzeqRvK7VaVjt4Morr6w7Xm8QVQ4qqmbRbAUqlZCjPI5VIHLI1zytDPQYNmxYaqM0o8Ew/F35nl47gwM0d2LUR1oFqr/AvIIMSAQyGVldNDg+ooAyrjORpBdVf6oKDKpUNwK6Cy1fvjy10b1H5XVeqwaKcZ2K8jJGsicDz3bdddfUxvFKmR/Ixp+ug+rqURWi+1iUhzKSTTmuGuWF5HzT36wIlZR57E7Jx7wHaP7USEqP7kE8r6LcnXqsyOWpqDqPXjfHip4nXTt0nPV1vmkg/7tGvxP7nP2gOcvnzp0LADjxxBPb/v6BAwem15z3DLgGsr5UN0B+JnpOibCl0xhjjDHGlE4lLJ3N7rxeeuklAHlLJ5/89cmbOyYNMKHVIqrTqnVJmYpAd1R0rtXgnVZhJSE9B60vHNWR5Tnr7peWWt2RMB0Sd3d0vAeynaRabbjj0xRC06dPB5CvXMC+1Vra0e5Wrcd9gf5WDDaIUmhxx13k2K5U0ZrSKrx2rRDCca9zhhYr/q46XmpTlgGxRSZyTK86mnKERCmTagOndLwUWW24ZlURBhdMmDAhtU2dOhVAPiUbFYyTTjoptXHtYIUSIFtv2Tc6BzmudCxxHNJCA2TWd10v+RntSwZ8cr0GssCHvqLRfSxKxUaKLJza71/5ylcA5IM7iui0pZ1BsEBW9U2t3QzQ0d+ZbbqmRBZbvo4swbXvdQePGyl7em/j+VWt4lCjMcT5wWpCWlWIRM8QjQK2+L4+W3E+qSLGdHisfghk6bFs6TTGGGOMMZXBD53GGGOMMaZ0KiGvNwuDhaJqGZrDjfny1NzLQAE1M9O8rMfjd0TBM9tuu23b567VjFj1Q6UpOu1qvtFHHnmk7lzuvvvu3Dnpa/6dylqU+aIgB71umtA11yb7IpKqNeDohhtuiC6514hyRkb5ztjWqPJFlJdS85dVlUiyWrt2LYB81Rf+nV4f5wWl4qgylgaPUbahxAZklXmUZgMk+prIlSKSpNimfcy+0j6LAm2qxgUXXAAgL0VyXrMyG5Dl+D3nnHPqjqGfZR+yH7T/+PvrvORYUzcoSuhLly5NbXSB0JzIXBv7WlLvjkhKL5oDn/70pwFklfMAYNq0aQDy846Bb5rblJ+NUDeoL37xiwCAr3/9640voBs0kI6/pV5jlNuZfaHuKtG9t7ZN51gU8Bm5SfHvoiBdbeNnhwwZEl9oxakNSgOKcyc3W2VJg0Xp/qd9zu/VOdvq2m5LpzHGGGOMKZ1KWDqjFDb6ZM6najqvqlUiCuGnk7BaP+n4rtZPWvJ0N0iLnwbP3H///bnzALLgGU0nUMQPf/jDutfq2LzjjjsCyDuN77///gDyu3lm/9d6t9xBNpvWJ9pl0nIVXbemlagS7L+owpDuvpup6as7Ru7m1fE9qqii71cV1sPWPuB41/HHv+OuVQPVaH3SHS3nnR636tbMIhoFKNSm/InS40SO+VrvuWrMnz8fQD6QiOsaAwYA4Fe/+hWAvLWfFckiyyXnSJReSscI12cNOqHio7W7Tz311Lo2Vl/R1E4rVqwIr7NsojUnspJT2aIFU4OgGLilteypUqi1nIrdJz7xiabO7eijj06v995776Y+U8See+6ZXvP3jtZa/U1pqdX7ZxScGKkIJKomR7Qtep/npAE2vM9rMC/7Z8mSJXXHqBqR5ZJtahmP+qMooE2V1uOOOw4AcPPNN6e2q6++GkC+3/Q5qxls6TTGGGOMMaXjh05jjDHGGFM6lZDXo6oNaj5mDkkG4LCiD5CZzNUkTxOxBv7QnK8yPJ3gVQbi8VRevPTSSwEAe+yxR2qLpKNW0bxzdJxX1wFWG9D+oTSqZvDIqZhEcmCUp5H9o/KxBj9VEfZVFBATUZSrrFEwEoPT+oOkrlBqKZKsgOxa+fvrexynGjSkeT+JBgr0Nxq5YHCcFMl3Or64flU5AG3UqFEA8nIcg3YYsAgAH/3oRwHkq6BFuThJlGcxWoeidYvfTxkPyGTzxx57LLU9/fTTAPKBj51AxwHPS92vimRhhe5cWh2O9zHKkVo5jOu/ziHei7QSFANfWJ1P0bHG7/rud7+b2nbeeWcAwJgxY1LbvffeW3ecIiJXOP39ivKCRlXf9B7EOcN7a6MgGaL9z+NxvQaye6WO1ej76cZRFJjVSSJ3nE4TBRfXvqcwUA3IXFc0D/ePf/xjAMCIESNSW6vPCbZ0GmOMMcaY0qmEpVOthtFOksEztGjpbjCyjHLHp1YpBhDpZ2nVUashrTp04gaAGTNmAAAuvPDC1KaWgFaJUkjwunXHQwdy3aVE6Uhqj9vOrinaCWmwUu3fNaoR2xsUWVs6cVzdBfcHImsmd/SqDkQVXgjbdB7S4qIVh5haQx3K+zONAoNqLeIaEBPVZef7WtO9agwfPhxA/rxpSdNKTbTM6TUzECMKJIvW5Aiuu2od47jS4ARa1TW9DS2JVL+AvCW0VYoCxKJ7ksJALFZzArJ7hgaurlq1CkDWTwyaAjJlTa3O7AO1NPF34fEB4Iwzzqj77MqVKwHk1zDe77TCTKtEn9UxwL6KKgJF1srIctoqRfXbgdj6GaWNU5WvN+jN+2ajuUgV909/+lNqmzdvHgDg8MMPT22HHnoogLz1n6pDs9jSaYwxxhhjSqfHls7I1ylK5MrdSJE1pjsWLlwIIEs/ojs6PnHrroFWHT0n7mIinxNti1IzMLG27pR6As81OhdNmUFLZyNLcG06l0Z1VqNjRf54UWLrVpPNlkmRP1nRrrlZXyF9L0qo3yjNTm8T+VjRSqTpkWhB2WKLLeqOQZ8ephMDsjRa0djTcaUpbUh/SaMUzY8ocXXR30eKRJUtnbw+VYR43mrR4liIfICjdGU8btR/0dxTqwmPp75lRMcr10StAd0TSyfXzEbr2uzZswEAJ554YmobPHgwgLw6RkujHo9/RyK1KFpfVKVQ6yihT92UKVPq3mOtdgCYNWsWgCzdFQAcc8wxAIBHH3207rMRZ599dnrN+5fOcVoV9bfibxnNmZ7AsaLrUhSvwHubjmmqN/osMXny5Lrz7CsVr1MUqQ5nnnlmes3fS1M7HnvssQDy1no+i+la30gJqMWWTmOMMcYYUzp+6DTGGGOMMaXTlrweyUg9kdHGjx+fXtMZm2k6gEwOpJlX5RjKLGo+5t/redLcrs7CNJ1HGfX1OxgsceSRR6a2m266qYkrKyaSb9XcH6V5Yj+r5F4rqxcFQOjfqRM1JbQorUVV4W8ZVQIpqjXfKPCoqMKIjouqpU+K5H5KcwzGAzLHb5XQeS2UAFUyYbUivV5K7pr2RaXO/sJOO+0EIP+7sh+jtGiR61AUhMK5o2mmqkZ0Lbx2rYIWpaWL0kQRtkUpk9SliOua9jOPq4FMHHe6xnMOR6m7mkWr6xx88MEAgJEjR6Y2ri86rllVR4Ms161bByBfzY2fLbrfRLXI9Rqj1EG8P+hc3GuvvQAAzzzzTN15quTPevU670844QQAeam1CAafAXE1QL5+8sknUxvHT1mytR6X65ZWP4qCizh+dOxxnevvkroSufmce+65APL3Qd4njjrqqNTG8aJ9xLlQlBqrEbZ0GmOMMcaY0mnL0tnI2ZpOqbpDZG1xbaPlkNYGIK7pzJ0h00rojo47PrVUMGWSWmu4u9NEptwNqaWVu0oNGuJT/bhx47q75LaIdlS6q43SI0UO57WfLaq3qp/V40YBOEVWjCoQWZhaDaZq5vhKu2k9+or99tsPQD7IglYItZYwaIyBCmq1oXVF59NWW21V911MX6NJqplmqaoBWLvssguAvEWI8z0KruPcajQ2uI5p8AhrbVet6IKuF/xtnnvuudSmNatriayktIxof0RW1aLE1VFwQqQMtZMubdCgQZg+fXpOuYqscTwHHQe8F+nf8T6i45pBr2oRrbVcqhWUx1OrIa9N+5+f0XPi3FVliqnPtI3Hacc6vM022wDIW0mjoEP2WRR0FlnKta1o/BB99oiCOznvdP3ifNb1juuc9o8Wk+kUzaYPa/e4Og75DKTKLQsCaLpHWjD1eufMmQMgvm9qURxauu+66662z71/3UGNMcYYY0y/xA+dxhhjjDGmdNqS11VmZg1YVpIAsmoRkeO3yg00bWv+LJrn1WxMeY+y1Kc+9an03rJlywDkJQOa2KMceaNHj06v+RnNqE/TtEoalE+iPIRlQklDK8ewH9UMHklXzaCyBCWIqDZyVWn1/KIgIxI5uevxI8fzKhDJ1iqbsL62yuucnxrkwjx9rBIzbNiw9B7nbJQjUGHAnVZL+f73v587t6rBajKN5lOt7BTJUDpe+L7m3T3ppJMAVEdeL3JB0TWHUm7kjhMFT0XuO9F31f69Hk/XX46/qGJMO1VkXnzxRfz85z/HPffck9ro+qD15bne672F+W51HYhkXt4P9b5Y6/4UBcRGkrJW/aJsr+4H7McoyFHb+FkNIF2wYEHd90XQTUfhdUe16VXKprudBp9wPBRVtmvHlYvfrxIzv0N/R/a3nmcZ97tIVo/uP61ea+RewmvmcwOQyea/+93vUhuf36ZNm9bUd0UBWFHwdbPY0mmMMcYYY0qnZbPNpptuiosvvjj9nwEFkYNvo1REUZogoo7A3HFecMEFdX9P60EUXHTbbbelNlp6GNAEZIFJUWWeyAqolSE6QaPdTZSyiP2n/V0bUBNZaCKHf3VG5+5XzykKpKhiIJH2RXS9tdaDRgFSRTWCdVxGFZt6m8iCyPq4QFbvWS1CPG9VApj2hY7nelwG2bAyF5AFmnAOAZl1THfaO+ywA4DmK570Ntz1qxUmctLn+CiydOu4YX+rJWWfffbpwBn3PrwWHRNRKrZaC06jecbXuv7yeGrp5NjRgIZIEWuFrq6uXBqxJUuW1P0Ng3rU6s/xrHOHwbFRYFCkRDAARy2YTAeoSiBfaxvvfY3urVG/8Htp8QSaX8+jFDlR0C+/l2qKvq/HiAKJaitZReNCKQpA03nHNq2SxM/2RVrAntxDawNlI0sqUyIB2XPR7rvvntqmT5/e0nfqd1Ada7UKkWJLpzHGGGOMKR0/dBpjjDHGmNJpSV4fOHAgJk2alAuooaO8VgDgazVnE5VsKVVqIA/NwZr7i1LelVdeCQCYPHlyeo+VgVTu4PePGTMmtR144IEA8mb6qOKPShSE5mU9dwZr6Ll3GsoXKiNQDohy60UBMLxGNelHFRoiuUYlkioSuUIUBQm166ytn9WxUlVUBr///vsB5McEx3h0LZFkxfGlUhjlKw1aomyvbgecl1WV13l+GjhTVG0nCuSL4N/pOsY8ptrvGtTR2zCAk8FjQCxjUupWSS0KBKl9L8qfGwVbqewaydJPPfUUAGDs2LGpLVobm+WNN97Ayy+/nLtuuolF64dWZ7r99tsB5KX0SHouCvjkZ6M5qe4b/Du9tzIwSYP6uA7qefA4Ov74e+vfMWevuhpE/P73v69ri8ZAFFTFe5aO9ah/eM5RbsvIlYp/F40B/X4eV/uW51S2u1h07+F9VXP4cvxxfHVH0fl+7WtfA5B3GeC9YMqUKYXHjdyGouqHnaiwZkunMcYYY4wpnZYsna+//jrWr1+fs+4xDYHuYvi+7tC4k9MdGneQWqeVn9FgIVpV+OR9/fXXp/dWrlwJIG/ppIVVd+Z0xtZdHo+nOzXuGiOnebWCsopSmZbOojQzRelcGln+IksE26IKFtFnqwB3YFGap56cZ+RcznFT5YpEnANaD53WEg1aYL8V/db6HsdhZBlVCzl37gxKAvIpY6oC094A2c5dK/Cwz6La4ZEDf5RiiWvFb37zm9TGFCWqwPR2+iRdw6LqZlFwXGRJqz2GHpt9E609UaqhKG1OVBNblaZIfWoVDajR17Xo3OD36e/Pe5bOj6JqVuzvaJ2JrHaaUpBKoPYt+0q/M7JSsU3nrAbgFnHYYYfVtfH+qvdZznedT3w/sj7qb8/zKwqI1euJFAn2QZQKKeqLTlcKqiW6DzGNXaQSqWW6mbREGrTJtF9qhY9SXRWdZ5FyAQDbbbddU8crorp3UGOMMcYY86bBD53GGGOMMaZ0WpLXX3vtNaxbty5nbmUOP3XKpmSl+cWYI0xzXdLcHckSaiKmhE9zOo8FALvssguAvDxCyVuDA/gd+lnKRSpzsE0lFQYAvPLKK6mNOeM0F2inKZJyi+TjZuV1/btIXldTfxWJgr4imaBdSVz7mOOiyn1C6UOvnXNM+4pzS6WlWkdylaAjqY6vH3/88dTGHLgqrTFYUIMKNTCjL9B8j1GAQuT8XxsEov1ZG8gHZH02cuTI1MY+45oF9L68HuXJ1N9VXSNI5LISyXBF+YL591EgiB6X72v1mIcffrjuPKPfTi5KAAAITUlEQVSKSGWhrl5RTmm9z7xZmThxYl0b10R1rePvxvzZADB37lwA+TlDlwEdR5Tho2o7HCPRGNTnB85PzafMICgNgNZnk1o0wEfXslq6uroaunFFa0kn5/xll12WXtPlL3KFaESRS4zOWeZx7gm2dBpjjDHGmNJpydL56quvYsWKFZg/f35qmzlzJoC8QzKr/6gzL52t1dmZ1kTdAXF3o7snPmlzt6AOtgyaiHbLujPmuWhwE3dWURWIKOBIK1MU7YCapdlgl0ZpQYpqikefjb6XlpzIElFVOG4iC1NPLCBRX3A8sCIJAKxYsaLt7yiD2kAFIJsraqHlHNQAgFprnc6TKOUJHdiXLVuW2saPHw8gH8jEOaiW0762dB5xxBHpNZUPne9Riij2B8eVrmO0rmgQDo9HlQTI+nH06NEduIqeEykekaWzKGBDP8vxV2QFjdbpaK6qperPf/5z3XdFwVumPLjWalAT1c3o99Zg30suuQQAMGPGjNRGi6hWNOMzRBSwGKkJXL80jQ/HlFaYuuiiiwAA+++/f93xonOfNGlSen355ZfXvU+auX9Hf8Mxu3DhwtTG9fT8889Pbddcc023xz3nnHMA5C3QvM5G6a9aRZ+jdB1vF1s6jTHGGGNM6fih0xhjjDHGlE5L8jpREzAlxtNPPz21MV+gBu1QttaAH8oxKq/XViUA6qUZlbb4Wo/BtqLqNEAmkauUyIAHNbtTImN1FyBzju4JRbk2gUw+aBS8wnNln2kwUKvVeBrJ61XK07n11lvXtUVSYG3/FLkX6N/r78M+1TFdNSgz6Vxg4N5uu+2W2iI5mJ/hdWogB99TdxlWuliwYEFq4xzX76ccE1W86CtGjBiRXvM6VQbnWFA3AL5Paf7mm29O7zG4JKr+olCO3HXXXXt2AR0iktdZ/UehW4UGgfL6olyTkWweyeF8HQWCaGAqJf8oMKlK4+rNDMeKrgtFwTjKWWedlfu3O/jb8zui+2Mkr0e5ZRvBY+v44TxW95vu5PXNN98cY8eOzbko8Tw0sIzPO+qaxHVU11OuSXPmzEltDFJev359ajvkkEMAALNnzwaQrxTVqH+bodG9Uc+5XWzpNMYYY4wxpdPyNnGTTTbJWQFvueWW3L9AVudcLaJMV6AO4nyCVosadx5RpQA+8evTOHfBupNg9ZVGljo6+2tgEs/p1ltvTW2rV68G0PvpTWrPCSi2ItT+CxSnFolSwihVDyTirkst31H9+dq0ONF1aSBJFBBBa7hWz6oatHTqb7lhwwYA+XnHOaYBP7ROcpeuikRRyimtdMTPar/xOKwtDAAPPfRQU9dTFmqlPOCAA+re5/lHFbn0egmtfWr1ILqOcbyyilpfEFkalchqREukWiQ5XzQVFq+1KJgvWrf0O2nhVBWD/RYpYlHaNNN5jj/+eADA1KlTUxst+9H9qR0iC2An0fRurJyk1lpaWu+8886Gx9pss80wdOjQXCVEHlOrLnKeqGrC9UWrGf7iF78AkFdTJ0yYACCrNARkChPPUS2jXH90nupzUbvo85FWWGsXWzqNMcYYY0zp+KHTGGOMMcaUTsvyepTXqpbFixcDAMaNG1f3nma0jyoXDRkyBADwxBNPpDaaqNesWdPq6VaaRkE5zFvGSgNAJl3p78DXlJmj9xrlMY3OqeqBREuXLgWQ758BAwYAiCuHUNrT4Iei61FJmH3G6ihVhC4AKodEedUoI6kczLFAiUiDRih58j0gm7salMOxFrl3aABCX6PBAazoobIvg8WitS5q49+rCwPXLL1uym7Mp9cX6Jzm76/zIXKluO666wDkZUO6OukaUhtUpO8VVSnSz7Hqm+Z/jY7P1+1WGzOtwXu0VvWhxKvjvii3ZESUezUKBiVRW3S/i4KQFi1alNroLqDzk0GR3/rWtxqe94YNG3DFFVc0/DvmIeVzDZC5pGgbz1f7l7K6niNze1599dUA8hI96YSkrqi7w2mnnQYAOO+889o+nmesMcYYY4wpnV7PN/Hggw8Wvt/pbPr9GVrtNH0IrQdahaF2h6iBNRFRQA13TJr2Ra1Ytd/VjMW7bGjRu+qqq1Ibg9i0f9h/UUopEjnDq+M5rfdqRawarH2u502rpsJr1d+au1kGy2n1EI45pvDQY2i/cbxqEBLPhf1XNVgdKAruiSwGW265ZV0bazVr4BH7TK0Uhx56KIC+DUbTc6R1JfoNFQ0I7UuiwMfofE15aEotBqzoGFfrHeH6q+sCiayUnUDvbVzvtYIclQhNl3jppZd27PsJAzn5b39EVedO9JEtncYYY4wxpnT80GmMMcYYY0rH5Rz6kEYViZYvXw4AWLVqVWqjQ3ckoVNy0lyCPG5UXUflDAYVaOAJA3WUKsjqhNekjs6aL5bQcZtVZTQggjz77LN1r6N8cY1+s75k1qxZAOLAkF/+8pepjW4TKvPWBvBFgRwKg0uUa6+9to2z7lvozqO/67777gsAGDVqVGo76KCDAMQ5/Cg5qfQ+b948APF47Es0XyCD4tauXZvalixZUveZRjl+ewvmMgSA4cOHAwDuu+++Xj+P/2V0LJxxxhkA8mNKc/+STge2NEM0PrWyDwNNNZiySve2qvLVr361x8ewpdMYY4wxxpROVys71q6urucBVLckS9+w/caNGwcB7p9ucP8Uk/oHcB91g8dQMR5DjfEYKsb9U4znWGNyfdQdLT10GmOMMcYY0w6W140xxhhjTOn4odMYY4wxxpSOHzqNMcYYY0zp+KHTGGOMMcaUjh86jTHGGGNM6fih0xhjjDHGlI4fOo0xxhhjTOn4odMYY4wxxpSOHzqNMcYYY0zp/B+Eg9VlY9QSawAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x1080 with 9 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(9, 15))\n",
    "fig.subplots_adjust(left=0, right=1, bottom=0, top=0.5, hspace=0.05, wspace=0.05)\n",
    "\n",
    "for i in range(9):\n",
    "    ax = fig.add_subplot(1, 9, i+1, xticks=[], yticks=[])\n",
    "    ax.imshow(x_train[i], cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.reshape((x_train.shape[0], 28, 28, 1)) / 255\n",
    "x_test = x_test.reshape((x_test.shape[0], 28, 28, 1)) / 255\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\suizh\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "# 28x28x1\n",
    "model.add(Conv2D(16, kernel_size=(5, 5), activation='relu', \n",
    "                 kernel_initializer='he_normal', input_shape=(28, 28, 1)))\n",
    "# 24x24x16\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# 12x12x16\n",
    "model.add(Conv2D(64, kernel_size=(5, 5), activation='relu', \n",
    "                 kernel_initializer='he_normal'))\n",
    "# 8x8x64\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# 4x4x64\n",
    "model.add(Flatten())\n",
    "# 1024\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "# 10\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 24, 24, 16)        416       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 12, 12, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 8, 8, 64)          25664     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 4, 4, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 10)                10250     \n",
      "=================================================================\n",
      "Total params: 36,330\n",
      "Trainable params: 36,330\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"543pt\" viewBox=\"0.00 0.00 378.00 543.00\" width=\"378pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 539)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-539 374,-539 374,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2651066103624 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2651066103624</title>\n",
       "<polygon fill=\"none\" points=\"41,-415.5 41,-461.5 329,-461.5 329,-415.5 41,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"97.5\" y=\"-434.8\">conv2d: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"154,-415.5 154,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"182\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"154,-438.5 210,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"182\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"210,-415.5 210,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"269.5\" y=\"-446.3\">(None, 28, 28, 1)</text>\n",
       "<polyline fill=\"none\" points=\"210,-438.5 329,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"269.5\" y=\"-423.3\">(None, 24, 24, 16)</text>\n",
       "</g>\n",
       "<!-- 2651066103512 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2651066103512</title>\n",
       "<polygon fill=\"none\" points=\"0.5,-332.5 0.5,-378.5 369.5,-378.5 369.5,-332.5 0.5,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"97.5\" y=\"-351.8\">max_pooling2d: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"194.5,-332.5 194.5,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"222.5\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"194.5,-355.5 250.5,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"222.5\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"250.5,-332.5 250.5,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"310\" y=\"-363.3\">(None, 24, 24, 16)</text>\n",
       "<polyline fill=\"none\" points=\"250.5,-355.5 369.5,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"310\" y=\"-340.3\">(None, 12, 12, 16)</text>\n",
       "</g>\n",
       "<!-- 2651066103624&#45;&gt;2651066103512 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2651066103624-&gt;2651066103512</title>\n",
       "<path d=\"M185,-415.366C185,-407.152 185,-397.658 185,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"188.5,-388.607 185,-378.607 181.5,-388.607 188.5,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2651128775288 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2651128775288</title>\n",
       "<polygon fill=\"none\" points=\"34,-249.5 34,-295.5 336,-295.5 336,-249.5 34,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"97.5\" y=\"-268.8\">conv2d_1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"161,-249.5 161,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"189\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"161,-272.5 217,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"189\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"217,-249.5 217,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"276.5\" y=\"-280.3\">(None, 12, 12, 16)</text>\n",
       "<polyline fill=\"none\" points=\"217,-272.5 336,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"276.5\" y=\"-257.3\">(None, 8, 8, 64)</text>\n",
       "</g>\n",
       "<!-- 2651066103512&#45;&gt;2651128775288 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>2651066103512-&gt;2651128775288</title>\n",
       "<path d=\"M185,-332.366C185,-324.152 185,-314.658 185,-305.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"188.5,-305.607 185,-295.607 181.5,-305.607 188.5,-305.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2651128775120 -->\n",
       "<g class=\"node\" id=\"node4\"><title>2651128775120</title>\n",
       "<polygon fill=\"none\" points=\"0,-166.5 0,-212.5 370,-212.5 370,-166.5 0,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"104.5\" y=\"-185.8\">max_pooling2d_1: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"209,-166.5 209,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"237\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"209,-189.5 265,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"237\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"265,-166.5 265,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"317.5\" y=\"-197.3\">(None, 8, 8, 64)</text>\n",
       "<polyline fill=\"none\" points=\"265,-189.5 370,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"317.5\" y=\"-174.3\">(None, 4, 4, 64)</text>\n",
       "</g>\n",
       "<!-- 2651128775288&#45;&gt;2651128775120 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>2651128775288-&gt;2651128775120</title>\n",
       "<path d=\"M185,-249.366C185,-241.152 185,-231.658 185,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"188.5,-222.607 185,-212.607 181.5,-222.607 188.5,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2651128883408 -->\n",
       "<g class=\"node\" id=\"node5\"><title>2651128883408</title>\n",
       "<polygon fill=\"none\" points=\"57,-83.5 57,-129.5 313,-129.5 313,-83.5 57,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"104.5\" y=\"-102.8\">flatten: Flatten</text>\n",
       "<polyline fill=\"none\" points=\"152,-83.5 152,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"180\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"152,-106.5 208,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"180\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"208,-83.5 208,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"260.5\" y=\"-114.3\">(None, 4, 4, 64)</text>\n",
       "<polyline fill=\"none\" points=\"208,-106.5 313,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"260.5\" y=\"-91.3\">(None, 1024)</text>\n",
       "</g>\n",
       "<!-- 2651128775120&#45;&gt;2651128883408 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>2651128775120-&gt;2651128883408</title>\n",
       "<path d=\"M185,-166.366C185,-158.152 185,-148.658 185,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"188.5,-139.607 185,-129.607 181.5,-139.607 188.5,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2651129086808 -->\n",
       "<g class=\"node\" id=\"node6\"><title>2651129086808</title>\n",
       "<polygon fill=\"none\" points=\"67,-0.5 67,-46.5 303,-46.5 303,-0.5 67,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"112\" y=\"-19.8\">dense: Dense</text>\n",
       "<polyline fill=\"none\" points=\"157,-0.5 157,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"185\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"157,-23.5 213,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"185\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"213,-0.5 213,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258\" y=\"-31.3\">(None, 1024)</text>\n",
       "<polyline fill=\"none\" points=\"213,-23.5 303,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258\" y=\"-8.3\">(None, 10)</text>\n",
       "</g>\n",
       "<!-- 2651128883408&#45;&gt;2651129086808 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>2651128883408-&gt;2651129086808</title>\n",
       "<path d=\"M185,-83.3664C185,-75.1516 185,-65.6579 185,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"188.5,-56.6068 185,-46.6068 181.5,-56.6069 188.5,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2650918272248 -->\n",
       "<g class=\"node\" id=\"node7\"><title>2650918272248</title>\n",
       "<polygon fill=\"none\" points=\"133,-498.5 133,-534.5 237,-534.5 237,-498.5 133,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"185\" y=\"-512.8\">2650918272248</text>\n",
       "</g>\n",
       "<!-- 2650918272248&#45;&gt;2651066103624 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2650918272248-&gt;2651066103624</title>\n",
       "<path d=\"M185,-498.254C185,-490.363 185,-480.749 185,-471.602\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"188.5,-471.591 185,-461.591 181.5,-471.591 188.5,-471.591\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "WARNING:tensorflow:From C:\\Users\\suizh\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/100\n",
      "60000/60000 [==============================] - 5s 83us/sample - loss: 0.5449 - acc: 0.8078 - val_loss: 0.4308 - val_acc: 0.8449\n",
      "Epoch 2/100\n",
      "60000/60000 [==============================] - 3s 57us/sample - loss: 0.3562 - acc: 0.8742 - val_loss: 0.3452 - val_acc: 0.8758\n",
      "Epoch 3/100\n",
      "60000/60000 [==============================] - 3s 58us/sample - loss: 0.3072 - acc: 0.8891 - val_loss: 0.3204 - val_acc: 0.8828\n",
      "Epoch 4/100\n",
      "60000/60000 [==============================] - 3s 58us/sample - loss: 0.2779 - acc: 0.9001 - val_loss: 0.2999 - val_acc: 0.8919\n",
      "Epoch 5/100\n",
      "60000/60000 [==============================] - 3s 57us/sample - loss: 0.2553 - acc: 0.9076 - val_loss: 0.2968 - val_acc: 0.8937\n",
      "Epoch 6/100\n",
      "60000/60000 [==============================] - 3s 57us/sample - loss: 0.2397 - acc: 0.9139 - val_loss: 0.2881 - val_acc: 0.8977\n",
      "Epoch 7/100\n",
      "60000/60000 [==============================] - 3s 57us/sample - loss: 0.2258 - acc: 0.9187 - val_loss: 0.2852 - val_acc: 0.8980\n",
      "Epoch 8/100\n",
      "60000/60000 [==============================] - 3s 57us/sample - loss: 0.2127 - acc: 0.9232 - val_loss: 0.2797 - val_acc: 0.8998\n",
      "Epoch 9/100\n",
      "60000/60000 [==============================] - 3s 58us/sample - loss: 0.2023 - acc: 0.9269 - val_loss: 0.2738 - val_acc: 0.8985\n",
      "Epoch 10/100\n",
      "60000/60000 [==============================] - 3s 57us/sample - loss: 0.1916 - acc: 0.9306 - val_loss: 0.2712 - val_acc: 0.9049\n",
      "Epoch 11/100\n",
      "60000/60000 [==============================] - 3s 58us/sample - loss: 0.1814 - acc: 0.9357 - val_loss: 0.2752 - val_acc: 0.9035\n",
      "Epoch 00011: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2694408abe0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "early_stopping = EarlyStopping(patience=1, verbose=1)\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=128, epochs=100, verbose=1,\n",
    "          validation_data=(x_test, y_test),\n",
    "          callbacks=[early_stopping])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.27522476626634595\n",
      "Test accuracy: 0.9035\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}