run_main_copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "# An implementation of https://arxiv.org/pdf/1512.03385.pdf                    #\n",
    "# See section 4.2 for the model architecture on CIFAR-10                       #\n",
    "# Some part of the code was referenced from below                              #\n",
    "# https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py   #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# Device configuration\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# Hyper-parameters\n",
    "num_epochs = 80\n",
    "learning_rate = 0.001\n",
    "\n",
    "# Image preprocessing modules\n",
    "transform = transforms.Compose([\n",
    "    transforms.Pad(4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.RandomCrop(32),\n",
    "    transforms.ToTensor()])\n",
    "\n",
    "# CIFAR-10 dataset\n",
    "train_dataset = torchvision.datasets.CIFAR10(root='./data/',\n",
    "                                             train=True, \n",
    "                                             transform=transform,\n",
    "                                             download=True)\n",
    "\n",
    "test_dataset = torchvision.datasets.CIFAR10(root='./data/',\n",
    "                                            train=False, \n",
    "                                            transform=transforms.ToTensor())\n",
    "\n",
    "# Data loader\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=100, \n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=100, \n",
    "                                          shuffle=False)\n",
    "\n",
    "# 3x3 convolution\n",
    "def conv3x3(in_channels, out_channels, stride=1):\n",
    "    return nn.Conv2d(in_channels, out_channels, kernel_size=3, \n",
    "                     stride=stride, padding=1, bias=False)\n",
    "\n",
    "# Residual block\n",
    "class ResidualBlock(nn.Module):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_planes, planes, stride=1,downsample=None):\n",
    "        super(ResidualBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(planes)\n",
    "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(planes)\n",
    "        self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)\n",
    "        self.bn3 = nn.BatchNorm2d(self.expansion*planes)\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_planes != self.expansion*planes:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(self.expansion*planes)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = F.relu(self.bn2(self.conv2(out)))\n",
    "        out = self.bn3(self.conv3(out))\n",
    "        out += self.shortcut(x)\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, num_blocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_planes = 16\n",
    "\n",
    "        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(16)\n",
    "        self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)\n",
    "        self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)\n",
    "        self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)\n",
    "        self.layer4 = self._make_layer(block, 128, num_blocks[3], stride=2)\n",
    "        self.linear = nn.Linear(128*block.expansion, num_classes)\n",
    "\n",
    "    def _make_layer(self, block, planes, num_blocks, stride):\n",
    "        strides = [stride] + [1]*(num_blocks-1)\n",
    "        layers = []\n",
    "        for stride in strides:\n",
    "            layers.append(block(self.in_planes, planes, stride))\n",
    "            self.in_planes = planes * block.expansion\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.layer4(out)\n",
    "        out = F.avg_pool2d(out, 4)\n",
    "        feature = out.view(out.size(0), -1)\n",
    "        out = self.linear(feature)\n",
    "        return out,feature\n",
    "    \n",
    "model = ResNet(ResidualBlock, [2, 2, 2, 2]).to(device)\n",
    "\n",
    "\n",
    "# Save the model checkpoint\n",
    "torch.save(model.state_dict(), 'resnet_copy1.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/80], Step [100/500] Loss: 1.6943\n",
      "Epoch [1/80], Step [200/500] Loss: 1.5973\n",
      "Epoch [1/80], Step [300/500] Loss: 1.4432\n",
      "Epoch [1/80], Step [400/500] Loss: 1.4038\n",
      "Epoch [1/80], Step [500/500] Loss: 1.2560\n",
      "Epoch [2/80], Step [100/500] Loss: 1.3023\n",
      "Epoch [2/80], Step [200/500] Loss: 1.2224\n",
      "Epoch [2/80], Step [300/500] Loss: 1.0166\n",
      "Epoch [2/80], Step [400/500] Loss: 1.0052\n",
      "Epoch [2/80], Step [500/500] Loss: 1.0240\n",
      "Epoch [3/80], Step [100/500] Loss: 1.1037\n",
      "Epoch [3/80], Step [200/500] Loss: 0.9200\n",
      "Epoch [3/80], Step [300/500] Loss: 1.1837\n",
      "Epoch [3/80], Step [400/500] Loss: 0.9647\n",
      "Epoch [3/80], Step [500/500] Loss: 0.8905\n",
      "Epoch [4/80], Step [100/500] Loss: 0.9064\n",
      "Epoch [4/80], Step [200/500] Loss: 0.8324\n",
      "Epoch [4/80], Step [300/500] Loss: 0.9925\n",
      "Epoch [4/80], Step [400/500] Loss: 0.9236\n",
      "Epoch [4/80], Step [500/500] Loss: 0.7773\n",
      "Epoch [5/80], Step [100/500] Loss: 0.9248\n",
      "Epoch [5/80], Step [200/500] Loss: 0.9864\n",
      "Epoch [5/80], Step [300/500] Loss: 0.9276\n",
      "Epoch [5/80], Step [400/500] Loss: 0.6165\n",
      "Epoch [5/80], Step [500/500] Loss: 0.7245\n",
      "Epoch [6/80], Step [100/500] Loss: 0.8012\n",
      "Epoch [6/80], Step [200/500] Loss: 0.6942\n",
      "Epoch [6/80], Step [300/500] Loss: 0.7370\n",
      "Epoch [6/80], Step [400/500] Loss: 0.6886\n",
      "Epoch [6/80], Step [500/500] Loss: 0.7466\n",
      "Epoch [7/80], Step [100/500] Loss: 0.6298\n",
      "Epoch [7/80], Step [200/500] Loss: 0.6733\n",
      "Epoch [7/80], Step [300/500] Loss: 0.6682\n",
      "Epoch [7/80], Step [400/500] Loss: 0.9021\n",
      "Epoch [7/80], Step [500/500] Loss: 0.7015\n",
      "Epoch [8/80], Step [100/500] Loss: 0.5157\n",
      "Epoch [8/80], Step [200/500] Loss: 0.7011\n",
      "Epoch [8/80], Step [300/500] Loss: 0.7922\n",
      "Epoch [8/80], Step [400/500] Loss: 0.6275\n",
      "Epoch [8/80], Step [500/500] Loss: 0.5699\n",
      "Epoch [9/80], Step [100/500] Loss: 0.4794\n",
      "Epoch [9/80], Step [200/500] Loss: 0.5968\n",
      "Epoch [9/80], Step [300/500] Loss: 0.5640\n",
      "Epoch [9/80], Step [400/500] Loss: 0.6202\n",
      "Epoch [9/80], Step [500/500] Loss: 0.5847\n",
      "Epoch [10/80], Step [100/500] Loss: 0.4357\n",
      "Epoch [10/80], Step [200/500] Loss: 0.6720\n",
      "Epoch [10/80], Step [300/500] Loss: 0.5189\n",
      "Epoch [10/80], Step [400/500] Loss: 0.4119\n",
      "Epoch [10/80], Step [500/500] Loss: 0.5564\n",
      "Epoch [11/80], Step [100/500] Loss: 0.5021\n",
      "Epoch [11/80], Step [200/500] Loss: 0.5093\n",
      "Epoch [11/80], Step [300/500] Loss: 0.5895\n",
      "Epoch [11/80], Step [400/500] Loss: 0.4847\n",
      "Epoch [11/80], Step [500/500] Loss: 0.5997\n",
      "Epoch [12/80], Step [100/500] Loss: 0.4447\n",
      "Epoch [12/80], Step [200/500] Loss: 0.7279\n",
      "Epoch [12/80], Step [300/500] Loss: 0.4837\n",
      "Epoch [12/80], Step [400/500] Loss: 0.5245\n",
      "Epoch [12/80], Step [500/500] Loss: 0.5594\n",
      "Epoch [13/80], Step [100/500] Loss: 0.4165\n",
      "Epoch [13/80], Step [200/500] Loss: 0.5133\n",
      "Epoch [13/80], Step [300/500] Loss: 0.3509\n",
      "Epoch [13/80], Step [400/500] Loss: 0.5034\n",
      "Epoch [13/80], Step [500/500] Loss: 0.4174\n",
      "Epoch [14/80], Step [100/500] Loss: 0.4542\n",
      "Epoch [14/80], Step [200/500] Loss: 0.5816\n",
      "Epoch [14/80], Step [300/500] Loss: 0.5408\n",
      "Epoch [14/80], Step [400/500] Loss: 0.5428\n",
      "Epoch [14/80], Step [500/500] Loss: 0.4218\n",
      "Epoch [15/80], Step [100/500] Loss: 0.5479\n",
      "Epoch [15/80], Step [200/500] Loss: 0.3429\n",
      "Epoch [15/80], Step [300/500] Loss: 0.5799\n",
      "Epoch [15/80], Step [400/500] Loss: 0.6763\n",
      "Epoch [15/80], Step [500/500] Loss: 0.5806\n",
      "Epoch [16/80], Step [100/500] Loss: 0.4224\n",
      "Epoch [16/80], Step [200/500] Loss: 0.4133\n",
      "Epoch [16/80], Step [300/500] Loss: 0.4340\n",
      "Epoch [16/80], Step [400/500] Loss: 0.3938\n",
      "Epoch [16/80], Step [500/500] Loss: 0.6311\n",
      "Epoch [17/80], Step [100/500] Loss: 0.3442\n",
      "Epoch [17/80], Step [200/500] Loss: 0.4249\n",
      "Epoch [17/80], Step [300/500] Loss: 0.4414\n",
      "Epoch [17/80], Step [400/500] Loss: 0.4081\n",
      "Epoch [17/80], Step [500/500] Loss: 0.5038\n",
      "Epoch [18/80], Step [100/500] Loss: 0.4209\n",
      "Epoch [18/80], Step [200/500] Loss: 0.4580\n",
      "Epoch [18/80], Step [300/500] Loss: 0.4830\n",
      "Epoch [18/80], Step [400/500] Loss: 0.4520\n",
      "Epoch [18/80], Step [500/500] Loss: 0.2578\n",
      "Epoch [19/80], Step [100/500] Loss: 0.4499\n",
      "Epoch [19/80], Step [200/500] Loss: 0.3932\n",
      "Epoch [19/80], Step [300/500] Loss: 0.3434\n",
      "Epoch [19/80], Step [400/500] Loss: 0.4817\n",
      "Epoch [19/80], Step [500/500] Loss: 0.3586\n",
      "Epoch [20/80], Step [100/500] Loss: 0.3311\n",
      "Epoch [20/80], Step [200/500] Loss: 0.4495\n",
      "Epoch [20/80], Step [300/500] Loss: 0.5334\n",
      "Epoch [20/80], Step [400/500] Loss: 0.4137\n",
      "Epoch [20/80], Step [500/500] Loss: 0.5278\n",
      "Epoch [21/80], Step [100/500] Loss: 0.3167\n",
      "Epoch [21/80], Step [200/500] Loss: 0.2620\n",
      "Epoch [21/80], Step [300/500] Loss: 0.3823\n",
      "Epoch [21/80], Step [400/500] Loss: 0.3472\n",
      "Epoch [21/80], Step [500/500] Loss: 0.3168\n",
      "Epoch [22/80], Step [100/500] Loss: 0.3833\n",
      "Epoch [22/80], Step [200/500] Loss: 0.2384\n",
      "Epoch [22/80], Step [300/500] Loss: 0.4481\n",
      "Epoch [22/80], Step [400/500] Loss: 0.3285\n",
      "Epoch [22/80], Step [500/500] Loss: 0.3706\n",
      "Epoch [23/80], Step [100/500] Loss: 0.3090\n",
      "Epoch [23/80], Step [200/500] Loss: 0.3350\n",
      "Epoch [23/80], Step [300/500] Loss: 0.3118\n",
      "Epoch [23/80], Step [400/500] Loss: 0.2909\n",
      "Epoch [23/80], Step [500/500] Loss: 0.2531\n",
      "Epoch [24/80], Step [100/500] Loss: 0.2869\n",
      "Epoch [24/80], Step [200/500] Loss: 0.1406\n",
      "Epoch [24/80], Step [300/500] Loss: 0.5651\n",
      "Epoch [24/80], Step [400/500] Loss: 0.3276\n",
      "Epoch [24/80], Step [500/500] Loss: 0.2341\n",
      "Epoch [25/80], Step [100/500] Loss: 0.3247\n",
      "Epoch [25/80], Step [200/500] Loss: 0.3105\n",
      "Epoch [25/80], Step [300/500] Loss: 0.3216\n",
      "Epoch [25/80], Step [400/500] Loss: 0.3710\n",
      "Epoch [25/80], Step [500/500] Loss: 0.3344\n",
      "Epoch [26/80], Step [100/500] Loss: 0.3443\n",
      "Epoch [26/80], Step [200/500] Loss: 0.2219\n",
      "Epoch [26/80], Step [300/500] Loss: 0.3605\n",
      "Epoch [26/80], Step [400/500] Loss: 0.2653\n",
      "Epoch [26/80], Step [500/500] Loss: 0.2408\n",
      "Epoch [27/80], Step [100/500] Loss: 0.3811\n",
      "Epoch [27/80], Step [200/500] Loss: 0.2255\n",
      "Epoch [27/80], Step [300/500] Loss: 0.2529\n",
      "Epoch [27/80], Step [400/500] Loss: 0.3002\n",
      "Epoch [27/80], Step [500/500] Loss: 0.3811\n",
      "Epoch [28/80], Step [100/500] Loss: 0.2541\n",
      "Epoch [28/80], Step [200/500] Loss: 0.3431\n",
      "Epoch [28/80], Step [300/500] Loss: 0.3420\n",
      "Epoch [28/80], Step [400/500] Loss: 0.4145\n",
      "Epoch [28/80], Step [500/500] Loss: 0.3399\n",
      "Epoch [29/80], Step [100/500] Loss: 0.2660\n",
      "Epoch [29/80], Step [200/500] Loss: 0.1838\n",
      "Epoch [29/80], Step [300/500] Loss: 0.1922\n",
      "Epoch [29/80], Step [400/500] Loss: 0.3336\n",
      "Epoch [29/80], Step [500/500] Loss: 0.2499\n",
      "Epoch [30/80], Step [100/500] Loss: 0.2821\n",
      "Epoch [30/80], Step [200/500] Loss: 0.3595\n",
      "Epoch [30/80], Step [300/500] Loss: 0.2610\n",
      "Epoch [30/80], Step [400/500] Loss: 0.2254\n",
      "Epoch [30/80], Step [500/500] Loss: 0.3648\n",
      "Epoch [31/80], Step [100/500] Loss: 0.2741\n",
      "Epoch [31/80], Step [200/500] Loss: 0.2925\n",
      "Epoch [31/80], Step [300/500] Loss: 0.2463\n",
      "Epoch [31/80], Step [400/500] Loss: 0.2849\n",
      "Epoch [31/80], Step [500/500] Loss: 0.2314\n",
      "Epoch [32/80], Step [100/500] Loss: 0.4248\n",
      "Epoch [32/80], Step [200/500] Loss: 0.3131\n",
      "Epoch [32/80], Step [300/500] Loss: 0.2667\n",
      "Epoch [32/80], Step [400/500] Loss: 0.2937\n",
      "Epoch [32/80], Step [500/500] Loss: 0.2711\n",
      "Epoch [33/80], Step [100/500] Loss: 0.3865\n",
      "Epoch [33/80], Step [200/500] Loss: 0.2225\n",
      "Epoch [33/80], Step [300/500] Loss: 0.1291\n",
      "Epoch [33/80], Step [400/500] Loss: 0.4703\n",
      "Epoch [33/80], Step [500/500] Loss: 0.2680\n",
      "Epoch [34/80], Step [100/500] Loss: 0.3063\n",
      "Epoch [34/80], Step [200/500] Loss: 0.2877\n",
      "Epoch [34/80], Step [300/500] Loss: 0.2566\n",
      "Epoch [34/80], Step [400/500] Loss: 0.3383\n",
      "Epoch [34/80], Step [500/500] Loss: 0.4267\n",
      "Epoch [35/80], Step [100/500] Loss: 0.3902\n",
      "Epoch [35/80], Step [200/500] Loss: 0.2784\n",
      "Epoch [35/80], Step [300/500] Loss: 0.2210\n",
      "Epoch [35/80], Step [400/500] Loss: 0.3420\n",
      "Epoch [35/80], Step [500/500] Loss: 0.2178\n",
      "Epoch [36/80], Step [100/500] Loss: 0.2635\n",
      "Epoch [36/80], Step [200/500] Loss: 0.2969\n",
      "Epoch [36/80], Step [300/500] Loss: 0.1986\n",
      "Epoch [36/80], Step [400/500] Loss: 0.2071\n",
      "Epoch [36/80], Step [500/500] Loss: 0.2446\n",
      "Epoch [37/80], Step [100/500] Loss: 0.2348\n",
      "Epoch [37/80], Step [200/500] Loss: 0.1889\n",
      "Epoch [37/80], Step [300/500] Loss: 0.4074\n",
      "Epoch [37/80], Step [400/500] Loss: 0.3108\n",
      "Epoch [37/80], Step [500/500] Loss: 0.2140\n",
      "Epoch [38/80], Step [100/500] Loss: 0.2399\n",
      "Epoch [38/80], Step [200/500] Loss: 0.3563\n",
      "Epoch [38/80], Step [300/500] Loss: 0.3768\n",
      "Epoch [38/80], Step [400/500] Loss: 0.2412\n",
      "Epoch [38/80], Step [500/500] Loss: 0.2905\n",
      "Epoch [39/80], Step [100/500] Loss: 0.1823\n",
      "Epoch [39/80], Step [200/500] Loss: 0.0989\n",
      "Epoch [39/80], Step [300/500] Loss: 0.1798\n",
      "Epoch [39/80], Step [400/500] Loss: 0.1600\n",
      "Epoch [39/80], Step [500/500] Loss: 0.1772\n",
      "Epoch [40/80], Step [100/500] Loss: 0.2224\n",
      "Epoch [40/80], Step [200/500] Loss: 0.1652\n",
      "Epoch [40/80], Step [300/500] Loss: 0.4062\n",
      "Epoch [40/80], Step [400/500] Loss: 0.1570\n",
      "Epoch [40/80], Step [500/500] Loss: 0.2778\n",
      "Epoch [41/80], Step [100/500] Loss: 0.2272\n",
      "Epoch [41/80], Step [200/500] Loss: 0.2877\n",
      "Epoch [41/80], Step [300/500] Loss: 0.1653\n",
      "Epoch [41/80], Step [400/500] Loss: 0.2297\n",
      "Epoch [41/80], Step [500/500] Loss: 0.2068\n",
      "Epoch [42/80], Step [100/500] Loss: 0.2503\n",
      "Epoch [42/80], Step [200/500] Loss: 0.2429\n",
      "Epoch [42/80], Step [300/500] Loss: 0.1516\n",
      "Epoch [42/80], Step [400/500] Loss: 0.2212\n",
      "Epoch [42/80], Step [500/500] Loss: 0.3004\n",
      "Epoch [43/80], Step [100/500] Loss: 0.2074\n",
      "Epoch [43/80], Step [200/500] Loss: 0.1514\n",
      "Epoch [43/80], Step [300/500] Loss: 0.2492\n",
      "Epoch [43/80], Step [400/500] Loss: 0.2171\n",
      "Epoch [43/80], Step [500/500] Loss: 0.2010\n",
      "Epoch [44/80], Step [100/500] Loss: 0.2638\n",
      "Epoch [44/80], Step [200/500] Loss: 0.1360\n",
      "Epoch [44/80], Step [300/500] Loss: 0.3496\n",
      "Epoch [44/80], Step [400/500] Loss: 0.1976\n",
      "Epoch [44/80], Step [500/500] Loss: 0.1614\n",
      "Epoch [45/80], Step [100/500] Loss: 0.2638\n",
      "Epoch [45/80], Step [200/500] Loss: 0.1822\n",
      "Epoch [45/80], Step [300/500] Loss: 0.2185\n",
      "Epoch [45/80], Step [400/500] Loss: 0.1585\n",
      "Epoch [45/80], Step [500/500] Loss: 0.1953\n",
      "Epoch [46/80], Step [100/500] Loss: 0.2602\n",
      "Epoch [46/80], Step [200/500] Loss: 0.2683\n",
      "Epoch [46/80], Step [300/500] Loss: 0.1996\n",
      "Epoch [46/80], Step [400/500] Loss: 0.1871\n",
      "Epoch [46/80], Step [500/500] Loss: 0.2544\n",
      "Epoch [47/80], Step [100/500] Loss: 0.1904\n",
      "Epoch [47/80], Step [200/500] Loss: 0.2113\n",
      "Epoch [47/80], Step [300/500] Loss: 0.1635\n",
      "Epoch [47/80], Step [400/500] Loss: 0.1151\n",
      "Epoch [47/80], Step [500/500] Loss: 0.2490\n",
      "Epoch [48/80], Step [100/500] Loss: 0.2535\n",
      "Epoch [48/80], Step [200/500] Loss: 0.2187\n",
      "Epoch [48/80], Step [300/500] Loss: 0.1991\n",
      "Epoch [48/80], Step [400/500] Loss: 0.1620\n",
      "Epoch [48/80], Step [500/500] Loss: 0.2227\n",
      "Epoch [49/80], Step [100/500] Loss: 0.1959\n",
      "Epoch [49/80], Step [200/500] Loss: 0.2605\n",
      "Epoch [49/80], Step [300/500] Loss: 0.2160\n",
      "Epoch [49/80], Step [400/500] Loss: 0.1983\n",
      "Epoch [49/80], Step [500/500] Loss: 0.2228\n",
      "Epoch [50/80], Step [100/500] Loss: 0.1591\n",
      "Epoch [50/80], Step [200/500] Loss: 0.1071\n",
      "Epoch [50/80], Step [300/500] Loss: 0.1843\n",
      "Epoch [50/80], Step [400/500] Loss: 0.1968\n",
      "Epoch [50/80], Step [500/500] Loss: 0.1610\n",
      "Epoch [51/80], Step [100/500] Loss: 0.1313\n",
      "Epoch [51/80], Step [200/500] Loss: 0.2870\n",
      "Epoch [51/80], Step [300/500] Loss: 0.0679\n",
      "Epoch [51/80], Step [400/500] Loss: 0.3119\n",
      "Epoch [51/80], Step [500/500] Loss: 0.2518\n",
      "Epoch [52/80], Step [100/500] Loss: 0.2505\n",
      "Epoch [52/80], Step [200/500] Loss: 0.2613\n",
      "Epoch [52/80], Step [300/500] Loss: 0.2763\n",
      "Epoch [52/80], Step [400/500] Loss: 0.2734\n",
      "Epoch [52/80], Step [500/500] Loss: 0.0951\n",
      "Epoch [53/80], Step [100/500] Loss: 0.2229\n",
      "Epoch [53/80], Step [200/500] Loss: 0.1839\n",
      "Epoch [53/80], Step [300/500] Loss: 0.3373\n",
      "Epoch [53/80], Step [400/500] Loss: 0.2424\n",
      "Epoch [53/80], Step [500/500] Loss: 0.1236\n",
      "Epoch [54/80], Step [100/500] Loss: 0.2897\n",
      "Epoch [54/80], Step [200/500] Loss: 0.1409\n",
      "Epoch [54/80], Step [300/500] Loss: 0.1668\n",
      "Epoch [54/80], Step [400/500] Loss: 0.1096\n",
      "Epoch [54/80], Step [500/500] Loss: 0.1905\n",
      "Epoch [55/80], Step [100/500] Loss: 0.2284\n",
      "Epoch [55/80], Step [200/500] Loss: 0.2042\n",
      "Epoch [55/80], Step [300/500] Loss: 0.1508\n",
      "Epoch [55/80], Step [400/500] Loss: 0.2999\n",
      "Epoch [55/80], Step [500/500] Loss: 0.2665\n",
      "Epoch [56/80], Step [100/500] Loss: 0.1101\n",
      "Epoch [56/80], Step [200/500] Loss: 0.1002\n",
      "Epoch [56/80], Step [300/500] Loss: 0.2157\n",
      "Epoch [56/80], Step [400/500] Loss: 0.1971\n",
      "Epoch [56/80], Step [500/500] Loss: 0.2264\n",
      "Epoch [57/80], Step [100/500] Loss: 0.1659\n",
      "Epoch [57/80], Step [200/500] Loss: 0.1064\n",
      "Epoch [57/80], Step [300/500] Loss: 0.1742\n",
      "Epoch [57/80], Step [400/500] Loss: 0.2751\n",
      "Epoch [57/80], Step [500/500] Loss: 0.1414\n",
      "Epoch [58/80], Step [100/500] Loss: 0.2994\n",
      "Epoch [58/80], Step [200/500] Loss: 0.1306\n",
      "Epoch [58/80], Step [300/500] Loss: 0.1931\n",
      "Epoch [58/80], Step [400/500] Loss: 0.1636\n",
      "Epoch [58/80], Step [500/500] Loss: 0.2257\n",
      "Epoch [59/80], Step [100/500] Loss: 0.1468\n",
      "Epoch [59/80], Step [200/500] Loss: 0.1697\n",
      "Epoch [59/80], Step [300/500] Loss: 0.1160\n",
      "Epoch [59/80], Step [400/500] Loss: 0.1186\n",
      "Epoch [59/80], Step [500/500] Loss: 0.1631\n",
      "Epoch [60/80], Step [100/500] Loss: 0.1938\n",
      "Epoch [60/80], Step [200/500] Loss: 0.2463\n",
      "Epoch [60/80], Step [300/500] Loss: 0.1691\n",
      "Epoch [60/80], Step [400/500] Loss: 0.0889\n",
      "Epoch [60/80], Step [500/500] Loss: 0.1225\n",
      "Epoch [61/80], Step [100/500] Loss: 0.0983\n",
      "Epoch [61/80], Step [200/500] Loss: 0.1801\n",
      "Epoch [61/80], Step [300/500] Loss: 0.2647\n",
      "Epoch [61/80], Step [400/500] Loss: 0.1597\n",
      "Epoch [61/80], Step [500/500] Loss: 0.1721\n",
      "Epoch [62/80], Step [100/500] Loss: 0.2514\n",
      "Epoch [62/80], Step [200/500] Loss: 0.1729\n",
      "Epoch [62/80], Step [300/500] Loss: 0.3015\n",
      "Epoch [62/80], Step [400/500] Loss: 0.1550\n",
      "Epoch [62/80], Step [500/500] Loss: 0.1566\n",
      "Epoch [63/80], Step [100/500] Loss: 0.1928\n",
      "Epoch [63/80], Step [200/500] Loss: 0.0916\n",
      "Epoch [63/80], Step [300/500] Loss: 0.1467\n",
      "Epoch [63/80], Step [400/500] Loss: 0.1563\n",
      "Epoch [63/80], Step [500/500] Loss: 0.1363\n",
      "Epoch [64/80], Step [100/500] Loss: 0.1017\n",
      "Epoch [64/80], Step [200/500] Loss: 0.1577\n",
      "Epoch [64/80], Step [300/500] Loss: 0.1285\n",
      "Epoch [64/80], Step [400/500] Loss: 0.1734\n",
      "Epoch [64/80], Step [500/500] Loss: 0.1745\n",
      "Epoch [65/80], Step [100/500] Loss: 0.1888\n",
      "Epoch [65/80], Step [200/500] Loss: 0.1408\n",
      "Epoch [65/80], Step [300/500] Loss: 0.1408\n",
      "Epoch [65/80], Step [400/500] Loss: 0.2072\n",
      "Epoch [65/80], Step [500/500] Loss: 0.1411\n",
      "Epoch [66/80], Step [100/500] Loss: 0.1724\n",
      "Epoch [66/80], Step [200/500] Loss: 0.1393\n",
      "Epoch [66/80], Step [300/500] Loss: 0.1547\n",
      "Epoch [66/80], Step [400/500] Loss: 0.1746\n",
      "Epoch [66/80], Step [500/500] Loss: 0.2824\n",
      "Epoch [67/80], Step [100/500] Loss: 0.1533\n",
      "Epoch [67/80], Step [200/500] Loss: 0.1400\n",
      "Epoch [67/80], Step [300/500] Loss: 0.2565\n",
      "Epoch [67/80], Step [400/500] Loss: 0.1205\n",
      "Epoch [67/80], Step [500/500] Loss: 0.1763\n",
      "Epoch [68/80], Step [100/500] Loss: 0.1201\n",
      "Epoch [68/80], Step [200/500] Loss: 0.1909\n",
      "Epoch [68/80], Step [300/500] Loss: 0.1371\n",
      "Epoch [68/80], Step [400/500] Loss: 0.0967\n",
      "Epoch [68/80], Step [500/500] Loss: 0.1852\n",
      "Epoch [69/80], Step [100/500] Loss: 0.2320\n",
      "Epoch [69/80], Step [200/500] Loss: 0.1091\n",
      "Epoch [69/80], Step [300/500] Loss: 0.1496\n",
      "Epoch [69/80], Step [400/500] Loss: 0.1786\n",
      "Epoch [69/80], Step [500/500] Loss: 0.2682\n",
      "Epoch [70/80], Step [100/500] Loss: 0.1936\n",
      "Epoch [70/80], Step [200/500] Loss: 0.1072\n",
      "Epoch [70/80], Step [300/500] Loss: 0.1075\n",
      "Epoch [70/80], Step [400/500] Loss: 0.1994\n",
      "Epoch [70/80], Step [500/500] Loss: 0.2665\n",
      "Epoch [71/80], Step [100/500] Loss: 0.1769\n",
      "Epoch [71/80], Step [200/500] Loss: 0.1471\n",
      "Epoch [71/80], Step [300/500] Loss: 0.2643\n",
      "Epoch [71/80], Step [400/500] Loss: 0.0997\n",
      "Epoch [71/80], Step [500/500] Loss: 0.1472\n",
      "Epoch [72/80], Step [100/500] Loss: 0.2576\n",
      "Epoch [72/80], Step [200/500] Loss: 0.1519\n",
      "Epoch [72/80], Step [300/500] Loss: 0.2073\n",
      "Epoch [72/80], Step [400/500] Loss: 0.1745\n",
      "Epoch [72/80], Step [500/500] Loss: 0.1572\n",
      "Epoch [73/80], Step [100/500] Loss: 0.0957\n",
      "Epoch [73/80], Step [200/500] Loss: 0.2038\n",
      "Epoch [73/80], Step [300/500] Loss: 0.1055\n",
      "Epoch [73/80], Step [400/500] Loss: 0.1223\n",
      "Epoch [73/80], Step [500/500] Loss: 0.3106\n",
      "Epoch [74/80], Step [100/500] Loss: 0.1052\n",
      "Epoch [74/80], Step [200/500] Loss: 0.1709\n",
      "Epoch [74/80], Step [300/500] Loss: 0.2037\n",
      "Epoch [74/80], Step [400/500] Loss: 0.1367\n",
      "Epoch [74/80], Step [500/500] Loss: 0.2152\n",
      "Epoch [75/80], Step [100/500] Loss: 0.1586\n",
      "Epoch [75/80], Step [200/500] Loss: 0.1676\n",
      "Epoch [75/80], Step [300/500] Loss: 0.2887\n",
      "Epoch [75/80], Step [400/500] Loss: 0.1817\n",
      "Epoch [75/80], Step [500/500] Loss: 0.1694\n",
      "Epoch [76/80], Step [100/500] Loss: 0.1426\n",
      "Epoch [76/80], Step [200/500] Loss: 0.2192\n",
      "Epoch [76/80], Step [300/500] Loss: 0.1922\n",
      "Epoch [76/80], Step [400/500] Loss: 0.2490\n",
      "Epoch [76/80], Step [500/500] Loss: 0.2064\n",
      "Epoch [77/80], Step [100/500] Loss: 0.2039\n",
      "Epoch [77/80], Step [200/500] Loss: 0.1651\n",
      "Epoch [77/80], Step [300/500] Loss: 0.1266\n",
      "Epoch [77/80], Step [400/500] Loss: 0.1808\n",
      "Epoch [77/80], Step [500/500] Loss: 0.1570\n",
      "Epoch [78/80], Step [100/500] Loss: 0.1893\n",
      "Epoch [78/80], Step [200/500] Loss: 0.1620\n",
      "Epoch [78/80], Step [300/500] Loss: 0.1895\n",
      "Epoch [78/80], Step [400/500] Loss: 0.2560\n",
      "Epoch [78/80], Step [500/500] Loss: 0.2232\n",
      "Epoch [79/80], Step [100/500] Loss: 0.2264\n",
      "Epoch [79/80], Step [200/500] Loss: 0.1790\n",
      "Epoch [79/80], Step [300/500] Loss: 0.1686\n",
      "Epoch [79/80], Step [400/500] Loss: 0.1993\n",
      "Epoch [79/80], Step [500/500] Loss: 0.0920\n",
      "Epoch [80/80], Step [100/500] Loss: 0.2594\n",
      "Epoch [80/80], Step [200/500] Loss: 0.1250\n",
      "Epoch [80/80], Step [300/500] Loss: 0.1515\n",
      "Epoch [80/80], Step [400/500] Loss: 0.1327\n",
      "Epoch [80/80], Step [500/500] Loss: 0.2100\n"
     ]
    }
   ],
   "source": [
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# For updating learning rate\n",
    "def update_lr(optimizer, lr):    \n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "curr_lr = learning_rate\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        # print(images)\n",
    "        # print(labels)\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs,feature = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print (\"Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}\"\n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "    # Decay learning rate\n",
    "    if (epoch+1) % 20 == 0:\n",
    "        curr_lr /= 3\n",
    "        update_lr(optimizer, curr_lr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the model on the test images: 87 %\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Test the model\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs,features = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "    print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}