lecture39.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture 39: ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "from torchvision import transforms,datasets, models\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "import torchvision\n",
    "\n",
    "import copy\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_transform = transforms.Compose([transforms.Scale(224),transforms.ToTensor()])\n",
    "BatchSize = 128\n",
    "\n",
    "trainset = datasets.CIFAR10(root='./CIFAR10', train=True, download=True, transform=apply_transform)\n",
    "trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,\n",
    "                                          shuffle=True, num_workers=4) # Creating dataloader\n",
    "\n",
    "testset = datasets.CIFAR10(root='./CIFAR10', train=False, download=True, transform=apply_transform)\n",
    "testLoader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,\n",
    "                                         shuffle=False, num_workers=4) # Creating dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Size of train and test datasets\n",
    "print('No. of samples in train set: '+str(len(trainLoader.dataset)))\n",
    "print('No. of samples in test set: '+str(len(testLoader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "net = models.resnet18()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Counting number of trainable parameters\n",
    "totalParams = 0\n",
    "for params in net.parameters():\n",
    "    print params.size()\n",
    "    totalParams += np.sum(np.prod(params.size()))\n",
    "print('Total number of parameters: '+str(totalParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modifying the last fully-connected layer for 10 classes\n",
    "net.fc = nn.Linear(512,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying initial weights for visualization\n",
    "init_weightConv1 = copy.deepcopy(net.conv1.weight.data) # 1st conv layer\n",
    "init_weightConv2 = copy.deepcopy(net.layer1[0].conv1.weight.data) # 2nd conv layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check availability of GPU\n",
    "use_gpu = torch.cuda.is_available()\n",
    "if use_gpu:\n",
    "    print('GPU is available!')   \n",
    "    net = net.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss() # Negative Log-likelihood\n",
    "optimizer = optim.Adam(net.parameters(), lr=1e-4) # Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "iterations = 10\n",
    "trainLoss = []\n",
    "trainAuxLoss = []\n",
    "trainTotalLoss = []\n",
    "trainAcc = []\n",
    "\n",
    "testLoss = []\n",
    "testAcc = []\n",
    "\n",
    "start = time.time()\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    runningLoss = 0.0   \n",
    "    avgTotalLoss = 0.0\n",
    "    running_correct = 0\n",
    "    net.train(True) # For training\n",
    "    for data in trainLoader:\n",
    "        # Initialize gradients to zero\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        inputs,labels = data\n",
    "        # Wrap them in Variable\n",
    "        if use_gpu:\n",
    "            inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n",
    "            # Feed-forward input data through the network        \n",
    "            outputs = net(inputs) \n",
    "            _, predicted = torch.max(outputs.data, 1)            \n",
    "            running_correct += (predicted.cpu() == labels.data.cpu()).sum()\n",
    "        else:\n",
    "            inputs, labels = Variable(inputs), Variable(labels)  \n",
    "            # Feed-forward input data through the network        \n",
    "            outputs = net(inputs) \n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            running_correct += (predicted == labels.data).sum()   \n",
    "        \n",
    "               \n",
    "        # Compute loss/error\n",
    "        loss = criterion(F.log_softmax(outputs), labels)\n",
    "        # Backpropagate loss and compute gradients\n",
    "        loss.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss += loss.data[0]         \n",
    "    avgTrainAcc = running_correct/50000.0\n",
    "    avgTrainLoss = runningLoss/50000.0    \n",
    "    trainAcc.append(avgTrainAcc)\n",
    "    trainLoss.append(avgTrainLoss)    \n",
    "    \n",
    "    # Evaluating performance on test set for each epoch\n",
    "    net.train(False) # For testing [Affects batch-norm and dropout layers (if any)]\n",
    "    running_correct = 0\n",
    "    for data in testLoader:\n",
    "        inputs,labels = data\n",
    "        # Wrap them in Variable\n",
    "        if use_gpu:\n",
    "            inputs, labels= Variable(inputs.cuda()), Variable(labels.cuda())\n",
    "            outputs = net(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1)            \n",
    "            running_correct += (predicted.cpu() == labels.data.cpu()).sum()\n",
    "        else:\n",
    "            inputs, labels = Variable(inputs), Variable(labels)\n",
    "            outputs = net(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            running_correct += (predicted == labels.data).sum()\n",
    "        \n",
    "        loss = criterion(F.log_softmax(outputs), labels)\n",
    "        runningLoss += loss.data[0]         \n",
    "    avgTestLoss = runningLoss/10000.0\n",
    "    avgTestAcc = running_correct/10000.0\n",
    "    testLoss.append(avgTestLoss)\n",
    "    testAcc.append(avgTestAcc)\n",
    "        \n",
    "    \n",
    "    # Plotting training loss vs Epochs\n",
    "    fig1 = plt.figure(1)        \n",
    "    plt.plot(range(epoch+1),trainLoss,'r-',label='train')  \n",
    "    plt.plot(range(epoch+1),testLoss,'g-',label='test') \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),trainAcc,'r-',label='train')    \n",
    "    plt.plot(range(epoch+1),testAcc,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Accuracy')    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('Iteration: {:.0f} /{:.0f}  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss,avgTestAcc*100,epochEnd//60,epochEnd%60))\n",
    "end = time.time()-start\n",
    "print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying trained weights for visualization\n",
    "if use_gpu:\n",
    "    trained_weightConv1 = copy.deepcopy(net.conv1.weight.data.cpu())\n",
    "    trained_weightConv2 = copy.deepcopy(net.layer1[0].conv1.weight.data.cpu())\n",
    "else:\n",
    "    trained_weightConv1 = copy.deepcopy(net.conv1.weight.data)\n",
    "    trained_weightConv2 = copy.deepcopy(net.layer1[0].conv1.weight.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Visualization of weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to show an image\n",
    "def imshow(img, strlabel):\n",
    "    npimg = img.numpy()\n",
    "    npimg = np.abs(npimg)\n",
    "    fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "    fig_size[0] = 10\n",
    "    fig_size[1] = 10\n",
    "    plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "    plt.figure()\n",
    "    plt.title(strlabel)\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Visualizing weights of 1st convolutional layer\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1,nrow=8,normalize=True),'Initial weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv1,nrow=8,normalize=True),'Trained weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1-trained_weightConv1,nrow=8,normalize=True),'Difference of weights: conv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Visualizing weights of 2nd convolutional layer\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1)-trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}