_03_minibatch_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from notebook2script_py.ipynb\n"
     ]
    }
   ],
   "source": [
    "#export\n",
    "import import_ipynb\n",
    "import notebook2script_py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted _02_fully_connected.ipynb to nb_02.py\n"
     ]
    }
   ],
   "source": [
    "notebook2script_py.notebook2script('_02_fully_connected.ipynb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uuMRQ0X0QTF7",
    "outputId": "a05ae6e5-8492-46de-e335-6b4ae05768c4"
   },
   "outputs": [],
   "source": [
    "#export\n",
    "from exp.nb_02 import *\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "l7B-LHiCNZSm"
   },
   "outputs": [],
   "source": [
    "#%load_ext autoreload\n",
    "#%autoreload 2\n",
    "\n",
    "#%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "3OcXnR_FPIT4"
   },
   "outputs": [],
   "source": [
    "mpl.rcParams['image.cmap'] = 'gray'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VTOQt0VnNSoK",
    "outputId": "437f3f1b-46c4-4cfd-a1da-85d41f462239"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\Anaconda\\Anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:498: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  ..\\torch\\csrc\\utils\\tensor_numpy.cpp:180.)\n",
      "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n",
      "F:\\Anaconda\\Anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:62: UserWarning: train_data has been renamed data\n",
      "  warnings.warn(\"train_data has been renamed data\")\n",
      "F:\\Anaconda\\Anaconda\\lib\\site-packages\\torchvision\\datasets\\mnist.py:52: UserWarning: train_labels has been renamed targets\n",
      "  warnings.warn(\"train_labels has been renamed targets\")\n"
     ]
    }
   ],
   "source": [
    "x_train , y_train , x_valid , y_valid = get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "33q0cs7RNzB6"
   },
   "outputs": [],
   "source": [
    "n , m = x_train.shape\n",
    "c = y_train.max() + 1\n",
    "nh = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "gAMQMVLhN2jT"
   },
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self , n_in , nh , n_out):\n",
    "        super().__init__()\n",
    "        self.layers = [nn.Linear(n_in , nh) , nn.ReLU() , nn.Linear(nh , n_out)]\n",
    "\n",
    "    def __call__(self , x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "IlS44e7cPHjI"
   },
   "outputs": [],
   "source": [
    "model = Model(m, nh, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "NApZNlm8PM-D"
   },
   "outputs": [],
   "source": [
    "pred = model(x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JtIoAj9CSgLu"
   },
   "source": [
    "**Cross entropy loss**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "dj4busPhQkRS"
   },
   "outputs": [],
   "source": [
    "def log_softmax(x): return (x.exp() / (x.exp().sum(-1 , keepdim = True))).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "eQFx2qccS5Tw"
   },
   "outputs": [],
   "source": [
    "sm_pred = log_softmax(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6mfkzxoxTCQW",
    "outputId": "7ba6c75c-cdd7-4a30-e950-981715ecd7f2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5., 0., 4.])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2i3rCqStTQLh",
    "outputId": "87e8d581-0381-4157-d96a-c8c4ffb58e3a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-2.2499, -2.3755, -2.1417], grad_fn=<IndexBackward>)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_pred[[0,1,2] , [5,0,4]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9QrGfazVTRuN",
    "outputId": "b204c8b7-a32b-4766-b0d8-c9cc4a68fd81"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "DGJv_lFHohy7"
   },
   "outputs": [],
   "source": [
    "y_train = y_train.type(torch.LongTensor)\n",
    "y_valid = y_valid.type(torch.LongTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "evs1qOF1TV1O"
   },
   "outputs": [],
   "source": [
    "def nll(input, target): return -input[torch.arange(input.size(0)) , target.type(torch.LongTensor)].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "8YU-lHqsTpgi"
   },
   "outputs": [],
   "source": [
    "loss = nll(sm_pred , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zdyTQOFNY1er",
    "outputId": "7986252e-29c9-4f5a-e85f-814af7b0e0d6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.3094, grad_fn=<NegBackward>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "4h0wqyYHmsT3"
   },
   "outputs": [],
   "source": [
    "def log_softmax(x): return x - x.exp().sum(-1 , keepdim = True).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "Plv-crAGjasT"
   },
   "outputs": [],
   "source": [
    "def logsumexp(x):\n",
    "    m = x.max(-1)[0]\n",
    "    return m + (x - m[ : , None]).exp().sum(-1).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "AougNY_OjprP"
   },
   "outputs": [],
   "source": [
    "def log_softmax(x): return x - x.logsumexp(-1 , keepdim = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-5lEcnQzm3ZN"
   },
   "source": [
    "**Basic training loop**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "k79LIeCJkhTA"
   },
   "outputs": [],
   "source": [
    "loss_func = F.cross_entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "O2Q_ww1Vm1mV"
   },
   "outputs": [],
   "source": [
    "#export\n",
    "def accuracy(out , yb): return (torch.argmax(out , dim = 1) == yb).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WuggDtqim1oi",
    "outputId": "fd26b556-26f0-43fc-f0cf-6fc197f66d34"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-0.0633,  0.0672,  0.1929, -0.1295,  0.1997,  0.0622, -0.1033, -0.1364,\n",
       "          0.0279, -0.0958], grad_fn=<SelectBackward>),\n",
       " torch.Size([64, 10]))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bs = 64                  # batch size\n",
    "\n",
    "xb = x_train[0 : bs]     # a mini-batch from x\n",
    "preds = model(xb)      # predictions\n",
    "preds[0] , preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PtiIM451m1qx",
    "outputId": "cf75ce39-6d3f-40e2-b85e-46ef5659b9c8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.3190, grad_fn=<NllLossBackward>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yb = y_train[0 : bs]\n",
    "loss_func(preds , yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-yWBiL2inMkY",
    "outputId": "d5dd6edb-b91d-4d3c-d203-d52527f775c6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0938)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(preds, yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "ZfgulNd8noN4"
   },
   "outputs": [],
   "source": [
    "lr = 0.5   # learning rate\n",
    "epochs = 1 # how many epochs to train for"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "pupfb8zhnp4E"
   },
   "outputs": [],
   "source": [
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        start_i = i * bs\n",
    "        end_i = start_i + bs\n",
    "        xb = x_train[start_i : end_i]\n",
    "        yb = y_train[start_i : end_i]\n",
    "        loss = loss_func(model(xb), yb)\n",
    "\n",
    "        loss.backward()\n",
    "    with torch.no_grad():\n",
    "        for l in model.layers:\n",
    "            if hasattr(l , 'weight'):\n",
    "                l.weight -= l.weight.grad * lr\n",
    "                l.bias   -= l.bias.grad * lr\n",
    "                l.weight.grad.zero_()\n",
    "                l.bias.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lsbhsP1woYyF",
    "outputId": "7736face-0739-4671-82d2-1c074e73db95"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(1153.0731, grad_fn=<NllLossBackward>), tensor(0.1875))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_func(model(xb) , yb) , accuracy(model(xb) , yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WpMzLXc7ovge"
   },
   "source": [
    "**Using parameters and optim**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oD7mSGEhqnY5"
   },
   "source": [
    "**Parameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "jmoMSIlMoqhM"
   },
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self, n_in, nh, n_out):\n",
    "        super().__init__()\n",
    "        self.l1 = nn.Linear(n_in,nh)\n",
    "        self.l2 = nn.Linear(nh,n_out)\n",
    "        \n",
    "    def __call__(self, x): return self.l2(F.relu(self.l1(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "lu3x3Wj7ow-C"
   },
   "outputs": [],
   "source": [
    "model = Model(m , nh , 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ExySCQxlpRM9",
    "outputId": "ac79c593-180c-4908-eb5c-625681dc0232"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l1: Linear(in_features=784, out_features=50, bias=True)\n",
      "l2: Linear(in_features=50, out_features=10, bias=True)\n"
     ]
    }
   ],
   "source": [
    "for name,l in model.named_children(): print(f\"{name}: {l}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TBhBDygSpZFK",
    "outputId": "747dd014-1279-4f28-885d-987c29bc8df7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (l1): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (l2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "5p94PnXUpeGC"
   },
   "outputs": [],
   "source": [
    "def fit():\n",
    "    for epoch in range(epochs):\n",
    "        for i in range((n - 1) // bs + 1):\n",
    "            start_i = i * bs\n",
    "            end_i = start_i + bs\n",
    "            xb = x_train[start_i : end_i]\n",
    "            yb = y_train[start_i : end_i]\n",
    "            loss = loss_func(model(xb), yb)\n",
    "\n",
    "            loss.backward()\n",
    "        with torch.no_grad():\n",
    "            for p in model.parameters():\n",
    "                p -= p.grad * lr\n",
    "                model.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oOapbW4hpgza",
    "outputId": "ec10f9d2-2b0e-42e5-d683-4bf3109e92d5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(29.7079, grad_fn=<NllLossBackward>), tensor(0.2500))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit()\n",
    "loss_func(model(xb) , yb) , accuracy(model(xb) , yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "j3kYr-Vkqtvm"
   },
   "source": [
    "**Registering modules**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "BSt3uRh4pg1k"
   },
   "outputs": [],
   "source": [
    "layers = [nn.Linear(m , nh) , nn.ReLU() , nn.Linear(nh , 10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "N3JjLiRWpg3f"
   },
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self , layers):\n",
    "        super().__init__()\n",
    "        self.layers = layers\n",
    "        for i , l in enumerate(self.layers):\n",
    "            self.add_module(f'layer_{i}' , l)\n",
    "\n",
    "    def __call__(self , x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "            return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "ZLkmKR3epg5f"
   },
   "outputs": [],
   "source": [
    "model = Model(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qZzDGAiErQRZ",
    "outputId": "fee652ef-ada8-4ea4-8a9d-b20f91891c09"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (layer_0): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (layer_1): ReLU()\n",
       "  (layer_2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ejOIbuZ3rTgG"
   },
   "source": [
    "**nn.ModuleList**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "Kc3ny6k9rRT_"
   },
   "outputs": [],
   "source": [
    "class SequentialModel(nn.Module):\n",
    "    def __init__(self , layers):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList(layers)\n",
    "\n",
    "    def __call__(self , x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "M4itQN7grrEu"
   },
   "outputs": [],
   "source": [
    "model = SequentialModel(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3mbUOUeyruow",
    "outputId": "cfb36460-803e-419d-dc98-77d0dc28b1b7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialModel(\n",
       "  (layers): ModuleList(\n",
       "    (0): Linear(in_features=784, out_features=50, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=50, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7K7NTyT6rvdM",
    "outputId": "d2fe7f44-771d-4294-8166-5d8f6b2682fc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(14.2915, grad_fn=<NllLossBackward>), tensor(0.3750))"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit()\n",
    "loss_func(model(xb) , yb) , accuracy(model(xb) , yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "suCGYORvr1YI"
   },
   "source": [
    "**nn.Sequential**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "wTfW0Wgzrx3Y"
   },
   "outputs": [],
   "source": [
    "model = nn.Sequential(nn.Linear(m , nh) , nn.ReLU() , nn.Linear(nh , 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yHhza8R-r6Nf",
    "outputId": "91da81a4-3c81-4d98-80ec-5da56d4b4ac5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(37.6614, grad_fn=<NllLossBackward>), tensor(0.3125))"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit()\n",
    "loss_func(model(xb) , yb) , accuracy(model(xb) , yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "u8RJZSKbsJeo"
   },
   "source": [
    "**optim**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "0VJEBZZSr_S7"
   },
   "outputs": [],
   "source": [
    "class Optimizer():\n",
    "    def __init__(self , params , lr = 0.5):\n",
    "        self.params , self.lr = list(params) , lr\n",
    "\n",
    "    def step(self):\n",
    "        with torch.no_grad():\n",
    "            for p in self.params:\n",
    "                p -= p.grad * self.lr\n",
    "\n",
    "    def zero_grad(self):\n",
    "        for p in self.params:\n",
    "            p.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "37UHnuvpstgZ"
   },
   "outputs": [],
   "source": [
    "model = nn.Sequential(nn.Linear(m , nh) , nn.ReLU() , nn.Linear(nh , 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "pUiT1Vl4svEd"
   },
   "outputs": [],
   "source": [
    "opt = Optimizer(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "id": "46CZ1DZGswwd"
   },
   "outputs": [],
   "source": [
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        start_i = i * bs\n",
    "        end_i = start_i + bs\n",
    "        xb = x_train[start_i:end_i]\n",
    "        yb = y_train[start_i:end_i]\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3lyl8n60tG_N",
    "outputId": "8de49214-92b9-497f-afdc-cbcbcc8b5858"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.1311, grad_fn=<NllLossBackward>), tensor(0.9375))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss , acc = loss_func(model(xb) , yb) , accuracy(model(xb) , yb)\n",
    "loss , acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "id": "CRgLmuSNtIXz"
   },
   "outputs": [],
   "source": [
    "#export\n",
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "id": "zoQvY3PbtOTM"
   },
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    model = nn.Sequential(nn.Linear(m , nh) , nn.ReLU() , nn.Linear(nh , 10))\n",
    "    return model , optim.SGD(model.parameters() , lr = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AdMj8h5dtQS3",
    "outputId": "a87cec42-0c2e-4450-fa9e-6ed6f984e5a0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.3368, grad_fn=<NllLossBackward>)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model , opt = get_model()\n",
    "loss_func(model(xb) , yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "l1YyomGRtXJP"
   },
   "outputs": [],
   "source": [
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        start_i = i * bs\n",
    "        end_i = start_i+bs\n",
    "        xb = x_train[start_i:end_i]\n",
    "        yb = y_train[start_i:end_i]\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred , yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UIwUY_RMtZ7-",
    "outputId": "97c983f4-303d-4967-9e36-4910a5866968"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.0805, grad_fn=<NllLossBackward>), tensor(0.9375))"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss , acc = loss_func(model(xb) , yb) , accuracy(model(xb) , yb)\n",
    "loss , acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8jjg_saZtjHG"
   },
   "source": [
    "**Dataset and DataLoader**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZtmpHvAMtk2R"
   },
   "source": [
    "**Dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "id": "tbYfX-UFtbWf"
   },
   "outputs": [],
   "source": [
    "#export\n",
    "class Dataset():\n",
    "    def __init__(self , x , y):\n",
    "        self.x , self.y = x , y\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.x)\n",
    "\n",
    "    def __getitem__(self , i):\n",
    "        return self.x[i] , self.y[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "id": "webHuZpQuErD"
   },
   "outputs": [],
   "source": [
    "train_ds , valid_ds = Dataset(x_train , y_train) , Dataset(x_valid , y_valid)\n",
    "assert len(train_ds) == len(x_train)\n",
    "assert len(valid_ds) == len(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "p3RwmFSduF5v",
    "outputId": "93829a62-f408-46e3-a4d4-b868fad08811"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.]]),\n",
       " tensor([5, 0, 4, 1, 9]))"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb,yb = train_ds[0 : 5]\n",
    "assert xb.shape == (5 , 28 * 28)\n",
    "assert yb.shape == (5 , )\n",
    "xb , yb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "id": "8Z4-tCgVuKbu"
   },
   "outputs": [],
   "source": [
    "model , opt = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "id": "pX3mMp2GuPal"
   },
   "outputs": [],
   "source": [
    "for epoch in range(epochs):\n",
    "    for i in range((n - 1) // bs + 1):\n",
    "        xb , yb = train_ds[i * bs : i * bs + bs]\n",
    "        pred = model(xb)\n",
    "        loss = loss_func(pred, yb)\n",
    "\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yFXmAn0Auxg2",
    "outputId": "054e22b9-e90c-4134-e0e5-8b8f80ab0042"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.2612, grad_fn=<NllLossBackward>), tensor(0.9375))"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss,acc = loss_func(model(xb) , yb) , accuracy(model(xb) , yb)\n",
    "assert acc > 0.7\n",
    "loss , acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eWhahIvKu_pV"
   },
   "source": [
    "**DataLoader**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "id": "lMy--4nNu6UU"
   },
   "outputs": [],
   "source": [
    "class DataLoader():\n",
    "    def __init__(self , ds , bs):\n",
    "        self.ds , self.bs = ds , bs\n",
    "\n",
    "    def __iter__(self):\n",
    "        for i in range(0 , len(self.ds) , self.bs):\n",
    "            yield self.ds[i : i + self.bs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "id": "G7CPgyHrvY8m"
   },
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds , bs)\n",
    "valid_dl = DataLoader(valid_ds , bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "W98iEr4Hvazm"
   },
   "outputs": [],
   "source": [
    "xb , yb = next(iter(valid_dl))\n",
    "assert xb.shape == (bs , 28 * 28)\n",
    "assert yb.shape == (bs ,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "id": "d2uB8rJ4wWTR"
   },
   "outputs": [],
   "source": [
    "model , opt = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "id": "OhFVlqzJw0OW"
   },
   "outputs": [],
   "source": [
    "def fit():\n",
    "    for epoch in range(epochs):\n",
    "        for xb , yb in train_dl:\n",
    "            pred = model(xb)\n",
    "            loss = loss_func(pred, yb)\n",
    "            loss.backward()\n",
    "            opt.step()\n",
    "            opt.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "s98DO0i-w9AD"
   },
   "outputs": [],
   "source": [
    "fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7sR_AC_5xEct",
    "outputId": "2b2f5465-4de8-4ba3-cf41-6c41fb1f7dc3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.1116, grad_fn=<NllLossBackward>), tensor(0.9688))"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss , acc = loss_func(model(xb) , yb) , accuracy(model(xb) , yb)\n",
    "assert acc > 0.7\n",
    "loss , acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p2E-5Wx-pMUL"
   },
   "source": [
    "**Random sampling**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "id": "I7IeTHMLxGGN"
   },
   "outputs": [],
   "source": [
    "class Sampler():\n",
    "    def __init__(self , ds , bs , shuffle = False):\n",
    "        self.n , self.bs , self.shuffle = len(ds) , bs , shuffle\n",
    "\n",
    "    def __iter__(self):\n",
    "        self.idxs = torch.randperm(self.n) if self.shuffle else torch.arange(self.n)\n",
    "        for i in range(0, self.n, self.bs): yield self.idxs[i:i+self.bs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "id": "D4jwR4T-py6z"
   },
   "outputs": [],
   "source": [
    "small_ds = Dataset(*train_ds[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dRab_j_dp0K4",
    "outputId": "23adb221-a686-4391-d1ba-08e16284dee3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([0, 1, 2]), tensor([3, 4, 5]), tensor([6, 7, 8]), tensor([9])]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = Sampler(small_ds,3,False)\n",
    "[o for o in s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FiAq2A62p1g9",
    "outputId": "6c266b64-d1c4-4da9-fcb4-003bffb6997a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([5, 4, 9]), tensor([3, 6, 7]), tensor([8, 0, 2]), tensor([1])]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = Sampler(small_ds,3,True)\n",
    "[o for o in s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "S4BObZoIp5A7"
   },
   "outputs": [],
   "source": [
    "def collate(b):\n",
    "    xs , ys = zip(*b)\n",
    "    return torch.stack(xs) , torch.stack(ys)\n",
    "\n",
    "class DataLoader():\n",
    "    def __init__(self , ds , sampler , collate_fn = collate):\n",
    "        self.ds , self.sampler , self.collate_fn = ds , sampler , collate_fn\n",
    "\n",
    "    def __iter__(self):\n",
    "        for s in self.sampler:\n",
    "            yield self.collate_fn([self.ds[i] for i in s])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "id": "J1e7RTVlrAiv"
   },
   "outputs": [],
   "source": [
    "train_samp = Sampler(train_ds , bs , shuffle = True)\n",
    "valid_samp = Sampler(valid_ds , bs , shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "id": "32-fmVrJrBy8"
   },
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds , sampler = train_samp , collate_fn = collate)\n",
    "valid_dl = DataLoader(valid_ds , sampler = valid_samp , collate_fn = collate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FaVNqXeerJab",
    "outputId": "dd530342-23f7-4844-e29e-f3d6e4a0812a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.1227, grad_fn=<NllLossBackward>), tensor(0.9531))"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model , opt = get_model()\n",
    "fit()\n",
    "\n",
    "loss ,acc = loss_func(model(xb) , yb) , accuracy(model(xb) , yb)\n",
    "assert acc > 0.7\n",
    "loss ,acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "id": "BQjbe026za2x"
   },
   "outputs": [],
   "source": [
    "#export\n",
    "from torch.utils.data import DataLoader, SequentialSampler, RandomSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_dls(train_ds, valid_ds, bs, **kwargs):\n",
    "    return (DataLoader(train_ds, batch_size=bs, shuffle=True, **kwargs),\n",
    "            DataLoader(valid_ds, batch_size=bs*2, **kwargs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reference for this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://github.com/fastai/course-v3/blob/master/nbs/dl2/03_minibatch_training.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "_03_minibatch_training.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}