Convolutional-Neural-network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/ZRC/miniconda3/envs/tryit/lib/python36.zip',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6/lib-dynload',\n",
       " '',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6/site-packages',\n",
       " '/Users/ZRC/miniconda3/envs/tryit/lib/python3.6/site-packages/IPython/extensions',\n",
       " '/Users/ZRC/.ipython',\n",
       " '/Users/ZRC']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"/Users/ZRC\")\n",
    "sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import RandomSampler\n",
    "\n",
    "\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "\n",
    "from torchsummary import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "batch_size = 64\n",
    "num_features = 28 * 28\n",
    "learning_rate = 0.1\n",
    "random_seed = 7\n",
    "num_epochs = 3\n",
    "\n",
    "# Architecture\n",
    "num_classes = 10\n",
    "device = torch.device(\"cuda: 0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "#         transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "    ]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = datasets.MNIST(root = \"data\", \n",
    "                               train = True, \n",
    "                               transform= data_transforms[\"train\"], \n",
    "                               download= True )\n",
    "\n",
    "val_dataset = datasets.MNIST(root = \"data\", \n",
    "                               train = False, \n",
    "                               transform= data_transforms[\"val\"], \n",
    "                               ) \n",
    "\n",
    "train_loader = DataLoader(dataset = train_dataset, \n",
    "                          batch_size=batch_size, \n",
    "                          shuffle= True)\n",
    "\n",
    "val_loader = DataLoader(dataset = val_dataset,\n",
    "                         batch_size=batch_size,\n",
    "                         shuffle= False)\n",
    "\n",
    "data_loader = {\"train\": train_loader, \"val\": val_loader}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNetZrc(torch.nn.Module):\n",
    "    def __init__(self, num_classes):\n",
    "        super(ConvNetZrc, self).__init__()\n",
    "        \n",
    "        # calculate same padding:\n",
    "        # (w - k + 2*p) / s + 1 = w   =>  p = ( s( w - 1) - w + k) / 2\n",
    "        \n",
    "        # [1*28*28] => [4*28*28]\n",
    "        # 28\n",
    "        self.conv_1 = torch.nn.Conv2d(in_channels = 1,\n",
    "                                    out_channels = 4,\n",
    "                                    kernel_size = (3,3),\n",
    "                                    stride = (1, 1),\n",
    "                                    padding = 1) # (1(28-1) - 28 + 3) / 2 = 1\n",
    "        \n",
    "        # [4*28*28] => [4*14*14]\n",
    "        self.pool_1 = torch.nn.MaxPool2d(kernel_size = (2,2),\n",
    "                                        stride = (2,2),\n",
    "                                        padding = 0) \n",
    "        \n",
    "        # [4*14*14] => [8*14*14]\n",
    "        self.conv_2 = torch.nn.Conv2d(in_channels = 4, \n",
    "                                     out_channels = 8,\n",
    "                                     kernel_size = (3,3),\n",
    "                                     stride = (1, 1),\n",
    "                                     padding = 1)\n",
    "        \n",
    "        # [8*14*14] => [8*7*7]\n",
    "        self.pool_2 = torch.nn.MaxPool2d(kernel_size = (2,2),\n",
    "                                        stride = (2,2),\n",
    "                                        padding = 0)\n",
    "        \n",
    "        self.fully_connected_layer = torch.nn.Linear(8*7*7, num_classes)\n",
    "        \n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.conv_1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.pool_1(x)\n",
    "        \n",
    "        x = self.conv_2(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.pool_2(x)\n",
    "        \n",
    "        logits = self.fully_connected_layer(x.view(-1, 8*7*7))\n",
    "        probas = F.softmax(logits, dim = 1)\n",
    "        \n",
    "        return logits,probas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(layer):\n",
    "    if isinstance(layer, torch.nn.Linear):\n",
    "        torch.nn.init.xavier_uniform_(layer.weight)\n",
    "#         torch.nn.init.normal_(layer.weight, 0.0, 0.1)\n",
    "        if layer.bias is not None:\n",
    "            torch.nn.init.constant_(layer.bias.data, 0)\n",
    "    elif isinstance(layer, torch.nn.Conv2d):\n",
    "        torch.nn.init.kaiming_normal_(layer.weight)\n",
    "        if layer.bias is not None:\n",
    "            torch.nn.init.constant_(layer.bias.data, 0) \n",
    "            \n",
    "model = ConvNetZrc(num_classes = num_classes)\n",
    "model.apply(init_weights)\n",
    "model = model.to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1            [-1, 4, 28, 28]              40\n",
      "         MaxPool2d-2            [-1, 4, 14, 14]               0\n",
      "            Conv2d-3            [-1, 8, 14, 14]             296\n",
      "         MaxPool2d-4              [-1, 8, 7, 7]               0\n",
      "            Linear-5                   [-1, 10]           3,930\n",
      "================================================================\n",
      "Total params: 4,266\n",
      "Trainable params: 4,266\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.04\n",
      "Params size (MB): 0.02\n",
      "Estimated Total Size (MB): 0.06\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(model, (1,28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(model, data_loader, device):\n",
    "    corrected_pred, num_examples = 0,0\n",
    "    model.eval()\n",
    "    for features, targets in data_loader:\n",
    "        features = features.to(device)\n",
    "        targets = targets.to(device)\n",
    "        logits, probas = model(features)\n",
    "        predicted_labels = torch.argmax(probas, dim=1)\n",
    "        num_examples += targets.size(0)\n",
    "        corrected_pred += torch.sum((predicted_labels == targets))\n",
    "    \n",
    "    return float(corrected_pred) / num_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, data_loader, optimizer, num_epochs, metric_func, random_seed = 7):\n",
    "    # Manual seed for deterministic data loader\n",
    "    torch.manual_seed(random_seed)\n",
    "    for epoch in range(num_epochs):\n",
    "        # set training mode\n",
    "        model.train() \n",
    "        for batch_idx, (features, targets) in enumerate(data_loader[\"train\"]):\n",
    "            features = features.to(device)\n",
    "            targets = targets.to(device)\n",
    "\n",
    "\n",
    "            ## forward pass\n",
    "            logits, probas = model(features)\n",
    "            loss = F.cross_entropy(logits,targets)\n",
    "\n",
    "            # backward pass\n",
    "            # clear the gradients of all tensors being optimized\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            ### Login\n",
    "            if not batch_idx % 50:\n",
    "                print ('Epoch: {0:03d}/{1:03d} | Batch {2:03d}/{3:03d} | Loss: {4:.2f}'.format(\n",
    "                    epoch+1, num_epochs, batch_idx, \n",
    "                         len(train_dataset)//batch_size, loss))\n",
    "\n",
    "        with torch.set_grad_enabled(False):\n",
    "            print('Epoch: {0:03d}/{1:03d} training accuracy: {2:.2f}'.format(\n",
    "                  epoch+1, num_epochs, \n",
    "                  metric_func(model, data_loader[\"train\"], device)))\n",
    "            \n",
    "            print('Epoch: {0:03d}/{1:03d} training accuracy: {2:.2f}'.format(\n",
    "                  epoch+1, num_epochs, \n",
    "                  metric_func(model, data_loader[\"val\"], device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/003 | Batch 000/937 | Loss: 0.16\n",
      "Epoch: 001/003 | Batch 050/937 | Loss: 0.09\n",
      "Epoch: 001/003 | Batch 100/937 | Loss: 0.11\n",
      "Epoch: 001/003 | Batch 150/937 | Loss: 0.12\n",
      "Epoch: 001/003 | Batch 200/937 | Loss: 0.20\n",
      "Epoch: 001/003 | Batch 250/937 | Loss: 0.05\n",
      "Epoch: 001/003 | Batch 300/937 | Loss: 0.07\n",
      "Epoch: 001/003 | Batch 350/937 | Loss: 0.13\n",
      "Epoch: 001/003 | Batch 400/937 | Loss: 0.13\n",
      "Epoch: 001/003 | Batch 450/937 | Loss: 0.08\n",
      "Epoch: 001/003 | Batch 500/937 | Loss: 0.06\n",
      "Epoch: 001/003 | Batch 550/937 | Loss: 0.10\n",
      "Epoch: 001/003 | Batch 600/937 | Loss: 0.03\n",
      "Epoch: 001/003 | Batch 650/937 | Loss: 0.02\n",
      "Epoch: 001/003 | Batch 700/937 | Loss: 0.03\n",
      "Epoch: 001/003 | Batch 750/937 | Loss: 0.14\n",
      "Epoch: 001/003 | Batch 800/937 | Loss: 0.05\n",
      "Epoch: 001/003 | Batch 850/937 | Loss: 0.03\n",
      "Epoch: 001/003 | Batch 900/937 | Loss: 0.08\n",
      "Epoch: 001/003 training accuracy: 0.97\n",
      "Epoch: 001/003 training accuracy: 0.97\n",
      "Epoch: 002/003 | Batch 000/937 | Loss: 0.03\n",
      "Epoch: 002/003 | Batch 050/937 | Loss: 0.04\n",
      "Epoch: 002/003 | Batch 100/937 | Loss: 0.17\n",
      "Epoch: 002/003 | Batch 150/937 | Loss: 0.02\n",
      "Epoch: 002/003 | Batch 200/937 | Loss: 0.13\n",
      "Epoch: 002/003 | Batch 250/937 | Loss: 0.15\n",
      "Epoch: 002/003 | Batch 300/937 | Loss: 0.01\n",
      "Epoch: 002/003 | Batch 350/937 | Loss: 0.19\n",
      "Epoch: 002/003 | Batch 400/937 | Loss: 0.03\n",
      "Epoch: 002/003 | Batch 450/937 | Loss: 0.21\n",
      "Epoch: 002/003 | Batch 500/937 | Loss: 0.11\n",
      "Epoch: 002/003 | Batch 550/937 | Loss: 0.03\n",
      "Epoch: 002/003 | Batch 600/937 | Loss: 0.06\n",
      "Epoch: 002/003 | Batch 650/937 | Loss: 0.07\n",
      "Epoch: 002/003 | Batch 700/937 | Loss: 0.06\n",
      "Epoch: 002/003 | Batch 750/937 | Loss: 0.04\n",
      "Epoch: 002/003 | Batch 800/937 | Loss: 0.07\n",
      "Epoch: 002/003 | Batch 850/937 | Loss: 0.04\n",
      "Epoch: 002/003 | Batch 900/937 | Loss: 0.04\n",
      "Epoch: 002/003 training accuracy: 0.97\n",
      "Epoch: 002/003 training accuracy: 0.97\n",
      "Epoch: 003/003 | Batch 000/937 | Loss: 0.06\n",
      "Epoch: 003/003 | Batch 050/937 | Loss: 0.05\n",
      "Epoch: 003/003 | Batch 100/937 | Loss: 0.07\n",
      "Epoch: 003/003 | Batch 150/937 | Loss: 0.02\n",
      "Epoch: 003/003 | Batch 200/937 | Loss: 0.13\n",
      "Epoch: 003/003 | Batch 250/937 | Loss: 0.07\n",
      "Epoch: 003/003 | Batch 300/937 | Loss: 0.05\n",
      "Epoch: 003/003 | Batch 350/937 | Loss: 0.06\n",
      "Epoch: 003/003 | Batch 400/937 | Loss: 0.02\n",
      "Epoch: 003/003 | Batch 450/937 | Loss: 0.19\n",
      "Epoch: 003/003 | Batch 500/937 | Loss: 0.26\n",
      "Epoch: 003/003 | Batch 550/937 | Loss: 0.01\n",
      "Epoch: 003/003 | Batch 600/937 | Loss: 0.02\n",
      "Epoch: 003/003 | Batch 650/937 | Loss: 0.12\n",
      "Epoch: 003/003 | Batch 700/937 | Loss: 0.05\n",
      "Epoch: 003/003 | Batch 750/937 | Loss: 0.07\n",
      "Epoch: 003/003 | Batch 800/937 | Loss: 0.01\n",
      "Epoch: 003/003 | Batch 850/937 | Loss: 0.04\n",
      "Epoch: 003/003 | Batch 900/937 | Loss: 0.06\n",
      "Epoch: 003/003 training accuracy: 0.97\n",
      "Epoch: 003/003 training accuracy: 0.97\n"
     ]
    }
   ],
   "source": [
    "train_model(model, data_loader, optimizer, num_epochs, metric_func = compute_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}