inception-resnet-v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
    "os.environ['CUDA_HOME'] = '/usr/local/cuda-7.5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.datasets import cifar10\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Lambda, BatchNormalization\n",
    "from keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D\n",
    "from keras.engine import merge, Input, Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import Callback, LearningRateScheduler, ModelCheckpoint, EarlyStopping\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.utils import np_utils\n",
    "import keras.backend as K\n",
    "import json\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "nb_classes = 10\n",
    "\n",
    "# the data, shuffled and split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# reorder dimensions for tensorflow\n",
    "x_train = np.transpose(x_train.astype('float32') / 255., (0, 2, 3, 1))\n",
    "x_test = np.transpose(x_test.astype('float32') / 255., (0, 2, 3, 1))\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# inception-resnet-v2\n",
    "\n",
    "http://arxiv.org/pdf/1602.07261v1.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# we reduce # filters by factor of 8 compared to original inception-v4\n",
    "nb_filters_reduction_factor = 8\n",
    "\n",
    "def inception_resnet_v2_stem(x):\n",
    "    # in original inception-resnet-v2, conv stride is 2\n",
    "    x = Convolution2D(32//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(x)\n",
    "    x = Convolution2D(32//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(x)\n",
    "    x = Convolution2D(64//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    # in original inception-resnet-v2, stride is 2\n",
    "    a = MaxPooling2D((3, 3), strides=(1, 1), border_mode='valid', dim_ordering='tf')(x)\n",
    "    # in original inception-resnet-v2, conv stride is 2\n",
    "    b = Convolution2D(96//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(x)\n",
    "    x = merge([a, b], mode='concat', concat_axis=-1)\n",
    "    \n",
    "    a = Convolution2D(64//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    a = Convolution2D(96//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(a)\n",
    "    b = Convolution2D(64//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(64//nb_filters_reduction_factor, 7, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    b = Convolution2D(64//nb_filters_reduction_factor, 1, 7, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    b = Convolution2D(96//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(b)\n",
    "    x = merge([a, b], mode='concat', concat_axis=-1)\n",
    "    \n",
    "    # in original inception-resnet-v2, conv stride should be 2\n",
    "    a = Convolution2D(192//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(x)\n",
    "    # in original inception-resnet-v2, stride is 2\n",
    "    b = MaxPooling2D((3, 3), strides=(1, 1), border_mode='valid', dim_ordering='tf')(x)\n",
    "    x = merge([a, b], mode='concat', concat_axis=-1)\n",
    "    \n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x\n",
    "\n",
    "\n",
    "def inception_resnet_v2_A(x):\n",
    "    shortcut = x\n",
    "    \n",
    "    a = Convolution2D(32//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    b = Convolution2D(32//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(32//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    \n",
    "    c = Convolution2D(32//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    c = Convolution2D(48//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(c)\n",
    "    c = Convolution2D(64//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(c)\n",
    "    \n",
    "    x = merge([a, b, c], mode='concat', concat_axis=-1)\n",
    "    x = Convolution2D(384//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='linear',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    x = merge([shortcut, x], mode='sum')\n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x\n",
    "\n",
    "\n",
    "def inception_resnet_v2_reduction_A(x):\n",
    "    a = MaxPooling2D((3, 3), strides=(2, 2), border_mode='valid', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(384//nb_filters_reduction_factor, 3, 3, subsample=(2, 2), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(x)\n",
    "    c = Convolution2D(256//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    c = Convolution2D(256//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(c)\n",
    "    c = Convolution2D(384//nb_filters_reduction_factor, 3, 3, subsample=(2, 2), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(c)\n",
    "    \n",
    "    x = merge([a, b, c], mode='concat', concat_axis=-1)\n",
    "    \n",
    "    return x\n",
    "    \n",
    "\n",
    "def inception_resnet_v2_B(x):\n",
    "    shortcut = x\n",
    "    \n",
    "    a = Convolution2D(192//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    b = Convolution2D(128//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(160//nb_filters_reduction_factor, 1, 7, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    b = Convolution2D(192//nb_filters_reduction_factor, 7, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    \n",
    "    x = merge([a, b], mode='concat', concat_axis=-1)\n",
    "    x = Convolution2D(1154//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='linear',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    x = merge([shortcut, x], mode='sum')\n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x\n",
    "\n",
    "\n",
    "def inception_resnet_v2_reduction_B(x):\n",
    "    a = MaxPooling2D((3, 3), strides=(2, 2), border_mode='valid', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(256//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(288//nb_filters_reduction_factor, 3, 3, subsample=(2, 2), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(b)\n",
    "    c = Convolution2D(256//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    c = Convolution2D(288//nb_filters_reduction_factor, 3, 3, subsample=(2, 2), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(c)\n",
    "    d = Convolution2D(256//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    d = Convolution2D(288//nb_filters_reduction_factor, 3, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(d)\n",
    "    d = Convolution2D(320//nb_filters_reduction_factor, 3, 3, subsample=(2, 2), activation='relu',\n",
    "                      init='he_normal', border_mode='valid', dim_ordering='tf')(d)\n",
    "    \n",
    "    x = merge([a, b, c, d], mode='concat', concat_axis=-1)\n",
    "    \n",
    "    return x\n",
    "\n",
    "\n",
    "def inception_resnet_v2_C(x):\n",
    "    shortcut = x\n",
    "    \n",
    "    a = Convolution2D(192//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    b = Convolution2D(192//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    b = Convolution2D(224//nb_filters_reduction_factor, 1, 3, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    b = Convolution2D(256//nb_filters_reduction_factor, 3, 1, subsample=(1, 1), activation='relu',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(b)\n",
    "    \n",
    "    x = merge([a, b], mode='concat', concat_axis=-1)\n",
    "    x = Convolution2D(2048//nb_filters_reduction_factor, 1, 1, subsample=(1, 1), activation='linear',\n",
    "                      init='he_normal', border_mode='same', dim_ordering='tf')(x)\n",
    "    \n",
    "    x = merge([shortcut, x], mode='sum')\n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.15 s, sys: 0 ns, total: 2.15 s\n",
      "Wall time: 1.06 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "img_rows, img_cols = 32, 32\n",
    "img_channels = 3\n",
    "\n",
    "# in original inception-resnet-v2, these are 5, 10, 5, respectively\n",
    "num_A_blocks = 1\n",
    "num_B_blocks = 1\n",
    "num_C_blocks = 1\n",
    "\n",
    "inputs = Input(shape=(img_rows, img_cols, img_channels))\n",
    "\n",
    "x = inception_resnet_v2_stem(inputs)\n",
    "for i in range(num_A_blocks):\n",
    "    x = inception_resnet_v2_A(x)\n",
    "x = inception_resnet_v2_reduction_A(x)\n",
    "for i in range(num_B_blocks):\n",
    "    x = inception_resnet_v2_B(x)\n",
    "x = inception_resnet_v2_reduction_B(x)\n",
    "for i in range(num_C_blocks):\n",
    "    x = inception_resnet_v2_C(x)\n",
    "\n",
    "x = AveragePooling2D(pool_size=(4, 4), strides=(1, 1), border_mode='valid', dim_ordering='tf')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = Flatten()(x)\n",
    "\n",
    "predictions = Dense(nb_classes, activation='softmax')(x)\n",
    "\n",
    "model = Model(input=inputs, output=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                       Output Shape        Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_3 (InputLayer)               (None, 32, 32, 3)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_79 (Convolution2D)   (None, 30, 30, 4)   112         input_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_80 (Convolution2D)   (None, 28, 28, 4)   148         convolution2d_79[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_81 (Convolution2D)   (None, 28, 28, 8)   296         convolution2d_80[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_82 (Convolution2D)   (None, 26, 26, 12)  876         convolution2d_81[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_9 (MaxPooling2D)      (None, 26, 26, 8)   0           convolution2d_81[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "merge_23 (Merge)                   (None, 26, 26, 20)  0           maxpooling2d_9[0][0]             \n",
      "                                                                   convolution2d_82[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_85 (Convolution2D)   (None, 26, 26, 8)   168         merge_23[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_86 (Convolution2D)   (None, 26, 26, 8)   456         convolution2d_85[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_83 (Convolution2D)   (None, 26, 26, 8)   168         merge_23[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_87 (Convolution2D)   (None, 26, 26, 8)   456         convolution2d_86[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_84 (Convolution2D)   (None, 24, 24, 12)  876         convolution2d_83[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_88 (Convolution2D)   (None, 24, 24, 12)  876         convolution2d_87[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "merge_24 (Merge)                   (None, 24, 24, 24)  0           convolution2d_84[0][0]           \n",
      "                                                                   convolution2d_88[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_89 (Convolution2D)   (None, 22, 22, 24)  5208        merge_24[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_10 (MaxPooling2D)     (None, 22, 22, 24)  0           merge_24[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "merge_25 (Merge)                   (None, 22, 22, 48)  0           convolution2d_89[0][0]           \n",
      "                                                                   maxpooling2d_10[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "activation_7 (Activation)          (None, 22, 22, 48)  0           merge_25[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_93 (Convolution2D)   (None, 22, 22, 4)   196         activation_7[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_91 (Convolution2D)   (None, 22, 22, 4)   196         activation_7[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_94 (Convolution2D)   (None, 22, 22, 6)   222         convolution2d_93[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_90 (Convolution2D)   (None, 22, 22, 4)   196         activation_7[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_92 (Convolution2D)   (None, 22, 22, 4)   148         convolution2d_91[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_95 (Convolution2D)   (None, 22, 22, 8)   440         convolution2d_94[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "merge_26 (Merge)                   (None, 22, 22, 16)  0           convolution2d_90[0][0]           \n",
      "                                                                   convolution2d_92[0][0]           \n",
      "                                                                   convolution2d_95[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_96 (Convolution2D)   (None, 22, 22, 48)  816         merge_26[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "merge_27 (Merge)                   (None, 22, 22, 48)  0           activation_7[0][0]               \n",
      "                                                                   convolution2d_96[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "activation_8 (Activation)          (None, 22, 22, 48)  0           merge_27[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_98 (Convolution2D)   (None, 22, 22, 32)  1568        activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_99 (Convolution2D)   (None, 22, 22, 32)  9248        convolution2d_98[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_100 (Convolution2D)  (None, 10, 10, 48)  13872       convolution2d_99[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_97 (Convolution2D)   (None, 10, 10, 48)  20784       activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_11 (MaxPooling2D)     (None, 10, 10, 48)  0           activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "merge_28 (Merge)                   (None, 10, 10, 144) 0           maxpooling2d_11[0][0]            \n",
      "                                                                   convolution2d_97[0][0]           \n",
      "                                                                   convolution2d_100[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_102 (Convolution2D)  (None, 10, 10, 16)  2320        merge_28[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_103 (Convolution2D)  (None, 10, 10, 20)  2260        convolution2d_102[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_101 (Convolution2D)  (None, 10, 10, 24)  3480        merge_28[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_104 (Convolution2D)  (None, 10, 10, 24)  3384        convolution2d_103[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "merge_29 (Merge)                   (None, 10, 10, 48)  0           convolution2d_101[0][0]          \n",
      "                                                                   convolution2d_104[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_105 (Convolution2D)  (None, 10, 10, 144) 7056        merge_29[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "merge_30 (Merge)                   (None, 10, 10, 144) 0           merge_28[0][0]                   \n",
      "                                                                   convolution2d_105[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "activation_9 (Activation)          (None, 10, 10, 144) 0           merge_30[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_110 (Convolution2D)  (None, 10, 10, 32)  4640        activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_106 (Convolution2D)  (None, 10, 10, 32)  4640        activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_108 (Convolution2D)  (None, 10, 10, 32)  4640        activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_111 (Convolution2D)  (None, 10, 10, 36)  10404       convolution2d_110[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_107 (Convolution2D)  (None, 4, 4, 36)    10404       convolution2d_106[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_109 (Convolution2D)  (None, 4, 4, 36)    10404       convolution2d_108[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_112 (Convolution2D)  (None, 4, 4, 40)    13000       convolution2d_111[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_12 (MaxPooling2D)     (None, 4, 4, 144)   0           activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "merge_31 (Merge)                   (None, 4, 4, 256)   0           maxpooling2d_12[0][0]            \n",
      "                                                                   convolution2d_107[0][0]          \n",
      "                                                                   convolution2d_109[0][0]          \n",
      "                                                                   convolution2d_112[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_114 (Convolution2D)  (None, 4, 4, 24)    6168        merge_31[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_115 (Convolution2D)  (None, 4, 4, 28)    2044        convolution2d_114[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_113 (Convolution2D)  (None, 4, 4, 24)    6168        merge_31[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_116 (Convolution2D)  (None, 4, 4, 32)    2720        convolution2d_115[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "merge_32 (Merge)                   (None, 4, 4, 56)    0           convolution2d_113[0][0]          \n",
      "                                                                   convolution2d_116[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_117 (Convolution2D)  (None, 4, 4, 256)   14592       merge_32[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "merge_33 (Merge)                   (None, 4, 4, 256)   0           merge_31[0][0]                   \n",
      "                                                                   convolution2d_117[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "activation_10 (Activation)         (None, 4, 4, 256)   0           merge_33[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "averagepooling2d_1 (AveragePooling2(None, 1, 1, 256)   0           activation_10[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)                (None, 1, 1, 256)   0           averagepooling2d_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)                (None, 256)         0           dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                    (None, 10)          2570        flatten_1[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 168220\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 404 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 40s - loss: 2.0607 - acc: 0.2159 - val_loss: 1.8525 - val_acc: 0.3041\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.8354 - acc: 0.3127 - val_loss: 1.7134 - val_acc: 0.3601\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.7324 - acc: 0.3545 - val_loss: 1.6312 - val_acc: 0.3951\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.6573 - acc: 0.3907 - val_loss: 1.5516 - val_acc: 0.4288\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.5826 - acc: 0.4239 - val_loss: 1.4792 - val_acc: 0.4590\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.5205 - acc: 0.4478 - val_loss: 1.4295 - val_acc: 0.4782\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.4689 - acc: 0.4726 - val_loss: 1.3860 - val_acc: 0.5030\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.4179 - acc: 0.4924 - val_loss: 1.3356 - val_acc: 0.5227\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.3752 - acc: 0.5111 - val_loss: 1.3363 - val_acc: 0.5266\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 39s - loss: 1.3316 - acc: 0.5280 - val_loss: 1.2930 - val_acc: 0.5369\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "nb_epoch = 10\n",
    "data_augmentation = True\n",
    "\n",
    "# Model saving callback\n",
    "#checkpointer = ModelCheckpoint(filepath='stochastic_depth_cifar10.hdf5', verbose=1, save_best_only=True)\n",
    "\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    history = model.fit(x_train, y_train, \n",
    "                        batch_size=batch_size, nb_epoch=nb_epoch, verbose=1,\n",
    "                        validation_data=(x_test, y_test), shuffle=True,\n",
    "                        callbacks=[])\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "\n",
    "    # realtime data augmentation\n",
    "    datagen_train = ImageDataGenerator(\n",
    "        featurewise_center=False,\n",
    "        samplewise_center=False,\n",
    "        featurewise_std_normalization=False,\n",
    "        samplewise_std_normalization=False,\n",
    "        zca_whitening=False,\n",
    "        rotation_range=0,\n",
    "        width_shift_range=0.125,\n",
    "        height_shift_range=0.125,\n",
    "        horizontal_flip=True,\n",
    "        vertical_flip=False)\n",
    "    datagen_train.fit(x_train)\n",
    "\n",
    "    # fit the model on the batches generated by datagen.flow()\n",
    "    history = model.fit_generator(datagen_train.flow(x_train, y_train, batch_size=batch_size, shuffle=True),\n",
    "                                  samples_per_epoch=x_train.shape[0], \n",
    "                                  nb_epoch=nb_epoch, verbose=1,\n",
    "                                  validation_data=(x_test, y_test),\n",
    "                                  callbacks=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}