train_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.4\n",
    "config.gpu_options.visible_device_list = \"0\"\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "from keras.datasets import cifar10\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "from .models import AttentionResNetCifar10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load data\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define generators for training and validation data\n",
    "train_datagen = ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True,\n",
    "    rotation_range=20,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    zoom_range=0.2,\n",
    "    horizontal_flip=True)\n",
    "\n",
    "val_datagen = ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True)\n",
    "\n",
    "# compute quantities required for featurewise normalization\n",
    "# (std, mean, and principal components if ZCA whitening is applied)\n",
    "train_datagen.fit(x_train)\n",
    "val_datagen.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# build a model\n",
    "model = AttentionResNetCifar10(n_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# plot model graph\n",
    "SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare usefull callbacks\n",
    "lr_reducer = ReduceLROnPlateau(monitor='val_acc', factor=0.2, patience=7, min_lr=10e-7, epsilon=0.01, verbose=1)\n",
    "early_stopper = EarlyStopping(monitor='val_acc', min_delta=0, patience=15, verbose=1)\n",
    "callbacks= [lr_reducer, early_stopper]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define loss, metrics, optimizer\n",
    "model.compile(keras.optimizers.Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "1562/1562 [==============================] - 304s 195ms/step - loss: 1.9663 - acc: 0.3429 - val_loss: 1.5818 - val_acc: 0.4564\n",
      "Epoch 2/200\n",
      "1562/1562 [==============================] - 458s 293ms/step - loss: 1.5777 - acc: 0.4458 - val_loss: 1.5661 - val_acc: 0.4970\n",
      "Epoch 3/200\n",
      "1562/1562 [==============================] - 428s 274ms/step - loss: 1.4027 - acc: 0.5012 - val_loss: 1.4493 - val_acc: 0.5338\n",
      "Epoch 4/200\n",
      "1562/1562 [==============================] - 429s 275ms/step - loss: 1.2705 - acc: 0.5515 - val_loss: 1.3063 - val_acc: 0.5899\n",
      "Epoch 5/200\n",
      "1562/1562 [==============================] - 438s 280ms/step - loss: 1.1694 - acc: 0.5890 - val_loss: 1.0526 - val_acc: 0.6367\n",
      "Epoch 6/200\n",
      "1562/1562 [==============================] - 460s 295ms/step - loss: 1.0992 - acc: 0.6160 - val_loss: 0.9479 - val_acc: 0.6646\n",
      "Epoch 7/200\n",
      "1562/1562 [==============================] - 428s 274ms/step - loss: 1.0190 - acc: 0.6436 - val_loss: 1.1057 - val_acc: 0.6377\n",
      "Epoch 8/200\n",
      "1562/1562 [==============================] - 429s 275ms/step - loss: 0.9617 - acc: 0.6668 - val_loss: 1.1163 - val_acc: 0.6409\n",
      "Epoch 9/200\n",
      "1562/1562 [==============================] - 434s 278ms/step - loss: 0.9123 - acc: 0.6830 - val_loss: 0.8517 - val_acc: 0.7115\n",
      "Epoch 10/200\n",
      "1562/1562 [==============================] - 462s 296ms/step - loss: 0.8733 - acc: 0.6987 - val_loss: 0.8275 - val_acc: 0.7177\n",
      "Epoch 11/200\n",
      "1562/1562 [==============================] - 428s 274ms/step - loss: 0.8415 - acc: 0.7084 - val_loss: 0.7908 - val_acc: 0.7366\n",
      "Epoch 12/200\n",
      "1562/1562 [==============================] - 428s 274ms/step - loss: 0.7984 - acc: 0.7228 - val_loss: 0.7677 - val_acc: 0.7441\n",
      "Epoch 13/200\n",
      "1562/1562 [==============================] - 433s 277ms/step - loss: 0.7754 - acc: 0.7332 - val_loss: 0.8360 - val_acc: 0.7210\n",
      "Epoch 14/200\n",
      "1562/1562 [==============================] - 465s 298ms/step - loss: 0.7537 - acc: 0.7393 - val_loss: 0.6472 - val_acc: 0.7765\n",
      "Epoch 15/200\n",
      "1562/1562 [==============================] - 432s 276ms/step - loss: 0.6382 - acc: 0.7789 - val_loss: 0.5923 - val_acc: 0.7991\n",
      "Epoch 21/200\n",
      "1562/1562 [==============================] - 427s 273ms/step - loss: 0.6216 - acc: 0.7847 - val_loss: 0.6145 - val_acc: 0.8005\n",
      "Epoch 22/200\n",
      "1562/1562 [==============================] - 470s 301ms/step - loss: 0.6113 - acc: 0.7881 - val_loss: 0.6474 - val_acc: 0.7905\n",
      "Epoch 23/200\n",
      "1562/1562 [==============================] - 434s 278ms/step - loss: 0.5973 - acc: 0.7919 - val_loss: 0.5679 - val_acc: 0.8063\n",
      "Epoch 24/200\n",
      "1562/1562 [==============================] - 429s 275ms/step - loss: 0.5802 - acc: 0.7992 - val_loss: 0.5876 - val_acc: 0.8114\n",
      "Epoch 25/200\n",
      "1562/1562 [==============================] - 429s 275ms/step - loss: 0.5716 - acc: 0.8024 - val_loss: 0.5431 - val_acc: 0.8206\n",
      "Epoch 26/200\n",
      "1562/1562 [==============================] - 468s 300ms/step - loss: 0.5544 - acc: 0.8078 - val_loss: 0.5473 - val_acc: 0.8135\n",
      "Epoch 27/200\n",
      "1562/1562 [==============================] - 429s 274ms/step - loss: 0.5503 - acc: 0.8098 - val_loss: 0.5849 - val_acc: 0.8130\n",
      "Epoch 28/200\n",
      "1562/1562 [==============================] - 399s 256ms/step - loss: 0.5421 - acc: 0.8124 - val_loss: 0.4928 - val_acc: 0.8370\n",
      "Epoch 29/200\n",
      "1562/1562 [==============================] - 435s 279ms/step - loss: 0.5341 - acc: 0.8158 - val_loss: 0.5054 - val_acc: 0.8307\n",
      "Epoch 30/200\n",
      "1562/1562 [==============================] - 410s 262ms/step - loss: 0.5227 - acc: 0.8198 - val_loss: 0.4690 - val_acc: 0.8467\n",
      "Epoch 31/200\n",
      "1562/1562 [==============================] - 402s 257ms/step - loss: 0.5087 - acc: 0.8250 - val_loss: 0.5475 - val_acc: 0.8228\n",
      "Epoch 32/200\n",
      "1562/1562 [==============================] - 469s 300ms/step - loss: 0.5029 - acc: 0.8248 - val_loss: 0.4678 - val_acc: 0.8452\n",
      "Epoch 33/200\n",
      "1562/1562 [==============================] - 451s 289ms/step - loss: 0.4940 - acc: 0.8287 - val_loss: 0.5783 - val_acc: 0.8103\n",
      "Epoch 34/200\n",
      "1562/1562 [==============================] - 455s 291ms/step - loss: 0.4922 - acc: 0.8283 - val_loss: 0.5398 - val_acc: 0.8206\n",
      "Epoch 35/200\n",
      "1562/1562 [==============================] - 454s 290ms/step - loss: 0.4810 - acc: 0.8342 - val_loss: 0.4586 - val_acc: 0.8452\n",
      "Epoch 36/200\n",
      "1562/1562 [==============================] - 489s 313ms/step - loss: 0.4765 - acc: 0.8345 - val_loss: 0.5029 - val_acc: 0.8380\n",
      "\n",
      "Epoch 00036: ReduceLROnPlateau reducing learning rate to 1.9999999494757503e-05.\n",
      "Epoch 37/200\n",
      "1562/1562 [==============================] - 487s 312ms/step - loss: 0.3686 - acc: 0.8715 - val_loss: 0.3988 - val_acc: 0.8723\n",
      "Epoch 38/200\n",
      "1562/1562 [==============================] - 526s 337ms/step - loss: 0.3428 - acc: 0.8791 - val_loss: 0.3837 - val_acc: 0.8786\n",
      "Epoch 39/200\n",
      "1562/1562 [==============================] - 459s 294ms/step - loss: 0.3330 - acc: 0.8830 - val_loss: 0.3626 - val_acc: 0.8865\n",
      "Epoch 40/200\n",
      "1562/1562 [==============================] - 474s 303ms/step - loss: 0.3273 - acc: 0.8846 - val_loss: 0.3984 - val_acc: 0.8735\n",
      "Epoch 41/200\n",
      "1562/1562 [==============================] - 477s 305ms/step - loss: 0.3183 - acc: 0.8878 - val_loss: 0.3770 - val_acc: 0.8755\n",
      "Epoch 42/200\n",
      "1562/1562 [==============================] - 385s 247ms/step - loss: 0.3165 - acc: 0.8894 - val_loss: 0.3783 - val_acc: 0.8773\n",
      "Epoch 43/200\n",
      "1562/1562 [==============================] - 309s 198ms/step - loss: 0.3060 - acc: 0.8934 - val_loss: 0.3710 - val_acc: 0.8813\n",
      "Epoch 44/200\n",
      "1562/1562 [==============================] - 305s 195ms/step - loss: 0.3042 - acc: 0.8923 - val_loss: 0.3605 - val_acc: 0.8841\n",
      "Epoch 45/200\n",
      "1562/1562 [==============================] - 437s 280ms/step - loss: 0.2955 - acc: 0.8972 - val_loss: 0.3608 - val_acc: 0.8828\n",
      "Epoch 46/200\n",
      "1562/1562 [==============================] - 445s 285ms/step - loss: 0.2951 - acc: 0.8965 - val_loss: 0.3875 - val_acc: 0.8791\n",
      "Epoch 47/200\n",
      "1562/1562 [==============================] - 449s 287ms/step - loss: 0.2906 - acc: 0.8980 - val_loss: 0.3521 - val_acc: 0.8874\n",
      "\n",
      "Epoch 00047: ReduceLROnPlateau reducing learning rate to 3.999999898951501e-06.\n",
      "Epoch 48/200\n",
      "1562/1562 [==============================] - 448s 287ms/step - loss: 0.2705 - acc: 0.9038 - val_loss: 0.3529 - val_acc: 0.8885\n",
      "Epoch 49/200\n",
      "1562/1562 [==============================] - 277s 177ms/step - loss: 0.2608 - acc: 0.9078 - val_loss: 0.3491 - val_acc: 0.8888\n",
      "Epoch 50/200\n",
      "1562/1562 [==============================] - 274s 175ms/step - loss: 0.2598 - acc: 0.9079 - val_loss: 0.3565 - val_acc: 0.8892\n",
      "Epoch 51/200\n",
      "1562/1562 [==============================] - 280s 179ms/step - loss: 0.2570 - acc: 0.9097 - val_loss: 0.3545 - val_acc: 0.8873\n",
      "Epoch 52/200\n",
      "1562/1562 [==============================] - 287s 184ms/step - loss: 0.2564 - acc: 0.9108 - val_loss: 0.3359 - val_acc: 0.8941\n",
      "Epoch 53/200\n",
      "1562/1562 [==============================] - 256s 164ms/step - loss: 0.2545 - acc: 0.9110 - val_loss: 0.3496 - val_acc: 0.8919\n",
      "Epoch 54/200\n",
      "1562/1562 [==============================] - 255s 163ms/step - loss: 0.2540 - acc: 0.9095 - val_loss: 0.3625 - val_acc: 0.8885\n",
      "\n",
      "Epoch 00054: ReduceLROnPlateau reducing learning rate to 1e-06.\n",
      "Epoch 55/200\n",
      "1562/1562 [==============================] - 255s 164ms/step - loss: 0.2453 - acc: 0.9135 - val_loss: 0.3466 - val_acc: 0.8947\n",
      "Epoch 56/200\n",
      "1562/1562 [==============================] - 257s 165ms/step - loss: 0.2457 - acc: 0.9130 - val_loss: 0.3451 - val_acc: 0.8918\n",
      "Epoch 57/200\n",
      "1562/1562 [==============================] - 255s 163ms/step - loss: 0.2523 - acc: 0.9103 - val_loss: 0.3498 - val_acc: 0.8904\n",
      "Epoch 58/200\n",
      "1562/1562 [==============================] - 254s 163ms/step - loss: 0.2510 - acc: 0.9113 - val_loss: 0.3442 - val_acc: 0.8946\n",
      "Epoch 59/200\n",
      "1562/1562 [==============================] - 257s 165ms/step - loss: 0.2465 - acc: 0.9139 - val_loss: 0.3636 - val_acc: 0.8863\n",
      "Epoch 60/200\n",
      "1562/1562 [==============================] - 256s 164ms/step - loss: 0.2429 - acc: 0.9141 - val_loss: 0.3313 - val_acc: 0.8956\n",
      "Epoch 61/200\n",
      "1562/1562 [==============================] - 248s 159ms/step - loss: 0.2423 - acc: 0.9135 - val_loss: 0.3470 - val_acc: 0.8915\n",
      "Epoch 62/200\n",
      "1562/1562 [==============================] - 255s 164ms/step - loss: 0.2458 - acc: 0.9127 - val_loss: 0.3557 - val_acc: 0.8894\n",
      "Epoch 63/200\n",
      "1562/1562 [==============================] - 254s 163ms/step - loss: 0.2417 - acc: 0.9137 - val_loss: 0.3481 - val_acc: 0.8926\n",
      "Epoch 64/200\n",
      "1562/1562 [==============================] - 250s 160ms/step - loss: 0.2409 - acc: 0.9146 - val_loss: 0.3340 - val_acc: 0.8952\n",
      "Epoch 65/200\n",
      "1562/1562 [==============================] - 248s 159ms/step - loss: 0.2414 - acc: 0.9143 - val_loss: 0.3490 - val_acc: 0.8928\n",
      "Epoch 66/200\n",
      "1562/1562 [==============================] - 244s 156ms/step - loss: 0.2450 - acc: 0.9139 - val_loss: 0.3551 - val_acc: 0.8929\n",
      "Epoch 67/200\n",
      "1562/1562 [==============================] - 244s 156ms/step - loss: 0.2478 - acc: 0.9130 - val_loss: 0.3348 - val_acc: 0.8917\n",
      "Epoch 68/200\n",
      "1562/1562 [==============================] - 243s 156ms/step - loss: 0.2448 - acc: 0.9129 - val_loss: 0.3511 - val_acc: 0.8917\n",
      "Epoch 69/200\n",
      "1562/1562 [==============================] - 296s 190ms/step - loss: 0.2454 - acc: 0.9137 - val_loss: 0.3487 - val_acc: 0.8926\n",
      "Epoch 70/200\n",
      "1562/1562 [==============================] - 434s 278ms/step - loss: 0.2404 - acc: 0.9147 - val_loss: 0.3518 - val_acc: 0.8894\n",
      "Epoch 71/200\n",
      "1562/1562 [==============================] - 439s 281ms/step - loss: 0.2452 - acc: 0.9125 - val_loss: 0.3399 - val_acc: 0.8927\n",
      "Epoch 72/200\n",
      "1562/1562 [==============================] - 445s 285ms/step - loss: 0.2391 - acc: 0.9173 - val_loss: 0.3542 - val_acc: 0.8916\n",
      "Epoch 73/200\n",
      "1562/1562 [==============================] - 448s 286ms/step - loss: 0.2414 - acc: 0.9150 - val_loss: 0.3471 - val_acc: 0.8924\n",
      "Epoch 74/200\n",
      "1562/1562 [==============================] - 444s 284ms/step - loss: 0.2413 - acc: 0.9156 - val_loss: 0.3429 - val_acc: 0.8942\n",
      "Epoch 75/200\n",
      "1562/1562 [==============================] - 438s 280ms/step - loss: 0.2429 - acc: 0.9151 - val_loss: 0.3553 - val_acc: 0.8895\n",
      "Epoch 00075: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f39db525e48>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fits the model on batches with real-time data augmentation\n",
    "batch_size = 32\n",
    "\n",
    "model.fit_generator(train_datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                    steps_per_epoch=len(x_train)//batch_size, epochs=200,\n",
    "                    validation_data=val_datagen.flow(x_test, y_test, batch_size=batch_size), \n",
    "                    validation_steps=len(x_test)//batch_size,\n",
    "                    callbacks=callbacks, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.3466754733324051, 0.8917]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate_generator(val_datagen.flow(x_test, y_test), steps=len(x_test)/32, use_multiprocessing=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}