Pruner Test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WMb2Tt6lCOM-"
   },
   "source": [
    "# Default Pytorch pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fastprogress\n",
    "import fastai\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.vision import *\n",
    "\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64\n",
    "epochs = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Jdj0MFnYQN28"
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 20, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(20, 50, 5, 1)\n",
    "        self.fc1 = nn.Linear(4*4*50, 500)\n",
    "        self.fc2 = nn.Linear(500, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = x.view(-1, 4*4*50)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.MNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (ImageList.from_folder(path)\n",
    "                .split_by_folder(train='training', valid='testing')\n",
    "                .label_from_folder()\n",
    "                .databunch()\n",
    "                .normalize())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABYkAAAWYCAYAAADzwUkSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xu4lmWZN/7rkkWxkSJlkYm7xg0Q9tNB2iiC9KY46qSIMJoN2lRmmEdaY1PoaONmIjVxyn1Zh9sZe0UoUwsr03AmRxPztcCUqTQYGtmZW8RY9/sH9XtnxvE+n1oPz7PWc30+x+Exk+e36z5BuHjW1zvIVVUlAAAAAADKtFW7FwAAAAAAoH2UxAAAAAAABVMSAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMGUxAAAAAAABVMSAwAAAAAUTElMU+Scx+ac78o5/ybnvDznfGS7dwI6X85595zzhpzzDe3eBehsOedtcs4Lc87P55yfyDkf2+6dgM6Vc74h57wq5/xMzvmxnPOH2r0T0Ll0OqSkJKYJcs5dKaVvpJRuSyltk1L6cErphpzzHm1dDCjBZSmlB9q9BFCEy1JKG1NKb0wpvS+ldEXOeVx7VwI62NyU0i5VVb0upXR4Sum8nPM+bd4J6EA6HX5PSUwzjEkpbZ9Suriqqk1VVd2VUvrnlNKs9q4FdLKc8zEppadTSt9r9y5AZ8s5D00pHZVSOrOqqueqqro3pXRr8lkH2EKqqvppVVUv/f4//u6vXdu4EtC5dDqklJTENEd+lb+3Z6sXAcqQc35dSumclNJft3sXoAh7pJQ2VVX12H/6ew+nlLxJDGwxOefLc84vpJQeTSmtSind0eaVgM6k0yGlpCSmOR5NKT2VUvpkznlgznlqSumAlNKQ9q4FdLBzU0pfqarqV+1eBCjC1iml3/y3v/eblNKwNuwCFKKqqpPS5ntmUkppQUrppfr/BsAfRadDSklJTBNUVfVySmlaSumwlNKv0+Y3+/53SmlFO/cCOlPOee+U0oEppYvbvQtQjOdSSq/7b3/vdSmlZ9uwC1CQ3/1Pv+9NKe2QUprd7n2AzqPT4fe62r0AnaGqqv+TNv+bppRSSjnnf0kpXdu+jYAONiWltEtK6cmcc0qb3/AbkHN+S1VV49u4F9C5HkspdeWcd6+q6vHf/b29Uko/beNOQFm6kt+TGNhCdDqklFKuqqrdO9ABcs7/X9r8BdRWKaWTUkofTSmN+U9/2AJAU+Sch6T/+kbfaWlzaTy7qqrVbVkK6Hg555vS5j846kMppb3T5t8bdL+qqhTFQFPlnEemlP5XSum2lNKLafP/gmpBSunYqqq+0c7dgM6k0yElv90EzTMrbf7DFJ5KKb07pXSQywTYEqqqeqGqql///q+0+X8GvkFBDGxhJ6WUBqfNn3X+KW3+F1MKYmBLqNLm31piRUppfUrp8ymlUxXEwBak08GbxAAAAAAAJfMmMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAULCuVj4s5+xPyYMOUFVVbvcOjXDnQGfoL3dOSu4d6BT95d5x50BncOcArfRqd443iQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACtbV7gUAAACAVxo2bFiYOfzww8PM5MmTw8zMmTNr5xMnTgzPWLZsWZgBoG/yJjEAAAAAQMGUxAAAAAAABVMSAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMGUxAAAAAAABVMSAwAAAAAUrKvdCwAAQCvss88+YWbevHm180mTJoVnVFUVZnLOvT7n3nvvDc9YuHBhmLnxxhtr56tXrw7PgNIMGTIkzOy///5hZubMmbXzD3zgAw3vtKXtvvvuYWbZsmUt2AT4n+y4445hZr/99qudn3HGGeEZ48aNa3inOqeddlrt/LLLLgvP2LhxY1N2YTNvEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAXLVVW17mE5t+5hwBZTVVVu9w6N6E93zsiRI8PMtddeG2YOOuig2nlXV1fDO0Ff0V/unJT6171TonvuuSfMTJw4sXaec/zDsZHP1804p1m7fOc736mdH3LIIeEZnaa/3DvunPa5/vrrw8yxxx4bZqKfx638ej3yyCOPhJl99tknzGzatKkZ63QUdw6RCRMmhJnbbrstzIwYMaJ2/uyzz4ZnLFq0KMy85S1v6XXmzDPPDM+YO3dumOGVXu3O8SYxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQsFxVVeselnPrHkafNWTIkDAzffr0MHPkkUfWzseMGROeceaZZ4aZBQsWhJnSVFWV271DI/rTnfOZz3wmzJxxxhm9fs573vOeMLNo0aJePweaqb/cOSn1r3unL+nu7g4zV155Ze182rRp4Rk5xz+Uos/GzTgjpZRmz54dZk444YTa+T777NOUXbbaqv69kaVLl4ZnjBs3Lsz0J/3l3nHntM/atWvDzPDhw8NMdKc06+v11atXh5lG7uLIrrvuGmZ++ctf9vo5ncadw4QJE2rnd9xxR3hGV1dXmLn22mtr548//nh4xuWXXx5mtt122zDz/e9/v3b+wgsvhGe8853vDDO80qvdOd4kBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIJ1tXsBOsvYsWPDzPz588PM6NGjw0zOuXZeVVV4xpw5c8LMggULwgz0FxdeeGGYWbRoUQs2aZ0pU6aEmQMOOCDMXH755bXz1atXN7oSFGX69Olh5qKLLgozO+20U+28kV/3Fy9eHGaWLVtWO//whz/c6zNSSulLX/pSmIk+g5xyyinhGY181unp6amdN/K57IEHHggzs2bNqp0/+uij4RnQVzzzzDNhZsCAAWHmySefrJ1/4QtfCM9YsmRJmHniiSfCzL333ls733nnncMzXn755TADpdlxxx3DzG233VY732abbcIzjjnmmDDTSB/TDGvXrg0z0eelo446Kjxj1KhRYWblypVhhs28SQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABSsq90L0Hd0d3eHmVNOOaV2PmfOnPCMnHOYWbBgQZg588wza+eTJk0KzzjhhBPCDHSS0047rd0rNNW+++4bZr74xS+GmTFjxoSZt7/97bXzww47LDwDOtHkyZNr5/Pnzw/PqKoqzESfHx599NHwjClTpoSZyOzZs3t9RqPWrFlTO48+C6WU0tChQ8PMxz/+8dp5T09PeMaECRPCzPve977aeSPfHugr9ttvvzAzYMCAMLNixYpmrBN6wxveEGaGDx9eO//hD38YnrFy5cqGd4JSNHJfjBgxonb+gx/8IDxj0aJFDe/UF/z0pz+tnR911FHhGUcccUSYufzyyxveqXTeJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAArW1e4FaI3u7u4wc8cdd4SZ8ePH186XLVsWnnHccceFmUbOedvb3lY7f+qpp8IzZs2aFWagGQ4++ODa+YknntiSPe68886WPKdZfvKTn9TOhw4dGp4xatSopuxy0EEHNeUc6DRz5sypnVdVFZ7RSGbhwoW1c7+m/88++9nPhpno16jRo0eHZ/T09ISZ6MfKmWeeGZ4BfcWqVavavcIf5Lzzzgszb3zjG2vn55xzTrPWgaIcccQRYSbnXDs/99xzwzOeffbZhnfqC8aNG1c7j75PUkpp/fr1zVqH5E1iAAAAAICiKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKFhXuxegObq7u2vnV155ZXjG+PHjw8ySJUtq54ccckh4xpo1a8JM9O1JKf423XPPPeEZX//618MMRAYNGhRm3vKWt9TOR44c2ax1aj3++ONh5vLLLw8zF154YTPWCW21Vf2/y+zp6WnJHo1o5Pv2Xe96V5hZsWJFM9aBpmjk88PUqVNr5znn8IxGPhvMmDEjzPBKjXzfzp07t3Z+7bXXhmc08s8Z+MPttddeYeboo48OMx/84Ad7vUsjX19BaaZNmxZmGvk5WlVV7fwnP/lJwzv1BY18mw844IDa+fLly8Mz/umf/qnhnYh5kxgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAAChYV7sXoDmuu+662vnUqVPDMxYsWBBmZs+eXTvv7u4Oz7j44ovDzP777x9mhgwZUjv/8pe/HJ4BzRD9vEgppfPPP7923tPT06x1ar35zW8OM9GuKbVu30hf2SOlxr5vL7jggjBz7LHHNmMdCI0dOzbMHHnkkWGmqqra+Zo1a8IzDjnkkDDDlrNs2bLaefTPuFHNOgf6i8MOOyzMRJ+7dtttt/CMgQMHNrxTnSVLltTOV61a1ZTnQCfZfvvtm3LOSy+9VDtv1dc9I0eODDOf//znw8wxxxwTZrbaqv691ajnovm8SQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABSsq90L0BwjRoyoneecwzMayVx11VW182nTpjXlOVVVhZnZs2fXzpcsWRKeAc1w4YUXhpmenp4WbNIcS5cuDTO33HJLCzaJ74tG7ooDDjggzEyePLnhnaC/GDJkSO18/vz54Rnd3d1hJvp52Mivx37Nbq81a9bUzteuXRue0ciPFSjNvHnzwszuu+9eO2/ks06zfOQjH6mdP/300y3aBMrz1FNP1c43btzYkj322muvMHPsscc25VnRnXLZZZc15Tk0zpvEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAULCudi9Aa1RVFWamTZsWZnLOvX5OI5YtWxZmFixY0JRnQW+ddtppYeaCCy5owSaxX/3qV2HmxBNPDDP/+q//2ox1WuKWW24JM1dddVWYecc73tHrXfbdd98wM2PGjNr5/Pnze70HZRg7dmztfPTo0eEZjfy6HmVmzZoVnkF7jRgxona+7bbbhmc08mOlkc930F8cdthhYWb33XcPM9HXV600atSo2vmDDz7Yok2g/7jpppvCzCWXXBJmdtxxx9r57NmzwzOee+65MLPnnnvWzv/iL/4iPKNZHn744dr5E0880aJN+D1vEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAXravcCNMfb3va2ljxn5513rp3ff//94Rnd3d1h5rjjjgsza9asCTPQCpdffnmYWbRoUQs2ib300kth5uc//3kLNmmdpUuXhpmVK1e2YJOURo0aFWbGjh3bgk0owemnn147zzmHZzSS+dKXvlQ79+t13xf9c27Wj5XFixc3vBP0dS+//HKY6enpCTPPPfdc7fzjH/94eMYhhxwSZo466qgwM2HChNr5rbfeGp4BpVm3bl2Y+exnPxtmos9tc+fObXin3thqq/hd0kbutkZcccUVTTmH5vEmMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAULCudi9A/zJp0qTa+bbbbhuesWzZsqZkoK/YuHFjmPFjum875phjwsxPfvKT2vm4ceOaskvOuSnnwLRp02rnVVU15TkLFy5syjm0z5gxY2rnfqzAK915551h5t3vfneYWbp0ae189erV4Rm33HJLmJk4cWKYAbaMuXPnhplf/OIXtfOpU6c2ZZfoa5rFixeHZ8ybNy/M7LXXXg3vRN/hTWIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgXe1egL6ju7s7zFx33XW186qqwjNmzJgRZl588cUwA9BKPT09tfNNmzb1+oyUUhozZkztfIcddgjPWLFiRZih8+Wce33Gk08+GWaWLFnS6+ew5Xz4wx8OM1deeWXtvJHPd3feeWdTMtBJ7rnnnpY855lnngkzv/jFL8LMlClTmrAN8N+98MILYearX/1qr+bNMnLkyDAzfPjwFmxCO3iTGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAArW1e4F6DuOPPLIMFNVVe182bJl4RmNZAD6miuuuKJ2fskllzTlOTNmzKid33LLLeEZK1asaMou9G/Rr9nRPKWU1qxZ05QM7dOMz3eN/Fj57Gc/2/BO0BuHHHJI7Xz8+PHhGStXrgwz11xzTaMrtd3QoUPDzKhRo8LMgw8+2Ix1gH6su7s7zOy8885h5plnngkz9913X0M70TreJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAArW1e4F6F9yzrXzhQsXtmgTgNa64ooraueXXHJJS/Y49dRTw8x9990XZlauXNmMdejDol+zGzF06NAwM3jw4Nr5iy++2Os9+J+dd955YWbq1Klhphmf7xYvXhxmILLDDjuEmfnz59fOBw0aFJ7xrW99K8xcc801Yaav+OAHPxhmdtpppzBz/PHHN2MdgLRhw4Yws2LFihZswh/Cm8QAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDButq9AK3R3d0dZk444YQwU1VV7XzhwoUN7wTAH+4d73hHmHnd614XZlauXNmMdejDol+zo3lKKY0ePTrMTJ8+vXZ+4403hmfwPzvjjDNq53PmzAnPaOSfc/T5bdasWeEZ0Azjx48PM4MGDer1c+66665en9EqRxxxRJiZO3duU5717LPPNuUcoP+aPHlymMk5h5mnn36617vstttuYWb58uW9fg7/jzeJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBd7V6A1thpp53CzPjx48PMkiVLaudPPvlkwzsBdJIBAwa0e4X/3yOPPBJmurp8BOh0a9eurZ1vu+224RlbbRW/T3DdddfVzseOHRuesXDhwjDz4IMPhplWGDJkSJhp5Nt8/fXXh5nRo0fXznPO4RmNfN/OmDEjzEArNPLr17PPPls7HzZsWHjGRz/60TBz66231s4HDhwYnvGmN70pzEQ//97//veHZ7z2ta8NM1/5ylfCTCPf/0BnO/zww8NMVVVh5qabbgoz++67b6/P2HnnncMMjfMmMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAULBcVVXrHpZz6x7Gf3HeeeeFmU9/+tNh5l3velftfPHixQ3vRP9VVVVu9w6NcOfQSnfccUeYOfDAA1uwSWNe85rXtHuFhvWXOyelvnXvHHzwwbXz22+/PTwj5/i7Pvos2YwzUkrpoYceCjORRx99NMyMGTOmdj5kyJDwjNGjR4eZZny/HH/88eEZCxYsCDMvvvhimClNf7l3+tKd0yqHH3547XzhwoVNec4LL7xQO+/q6grPaOTX2uguaOR+/NnPfhZmpkyZEmaeeuqpMMOW4c6hr9i0aVOYaeReiu7QlOJ79OGHHw7P2HfffcMMr/Rqd443iQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgILlqqpa97CcW/cw/otG/jn39PSEmQEDBjRjHfq5qqpyu3dohDuHVho3blyYeeihh1qwSWNe85rXtHuFhvWXOyelzrt3pk+fHmYuuuii2vkuu+wSntHIZ5Cc638YNPJZJzqjkXMaOWP16tVhZsmSJWFm1qxZtfM1a9aEZ/DH6S/3TqfdOY2Ifg5ec8014Rl/+Zd/2aRtei/69nz/+98Pz4juipRS+vd///eGd6L13Dn0FZs2bQozzeoRly5dWjs/9dRTwzPuuuuupuxSmle7c7xJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAXLVVW17mE5t+5hhZk+fXrt/Oabbw7PWLBgQZiZOXNmwzvRuaqqyu3eoRHuHFpp2LBhYWbq1KlhZsaMGbXzo446KjzjtNNOCzNf/OIXw0xf0V/unJTKvHdGjBhRO99pp53CM4488sheP+fRRx8NzxgzZkyYiXz5y18OM2vWrAkzTz75ZK93YcvpL/dOiXcOdCJ3Dn3FjTfeGGaOPvroMPPAAw+EmZNOOql2/tBDD4Vn8Md5tTvHm8QAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBclVVrXtYzq17WGFOPPHE2vmnP/3p8Iy3ve1tYWbNmjUN70Tnqqoqt3uHRrhz6I+6u7tr5yNGjAjP+Ld/+7cws3HjxoZ3arf+cuek5N6BTtFf7h13DnQGdw7QSq9253iTGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKFhXuxegOZYuXVo7X7NmTXhGIxkAtqzVq1f3ag4AAAB/KG8SAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMGUxAAAAAAABVMSAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMFyVVWte1jOrXsYsMVUVZXbvUMj3DnQGfrLnZOSewc6RX+5d9w50BncOUArvdqd401iAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYLmqqnbvAAAAAABAm3iTGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpimiLnfEPOeVXO+Zmc82M55w+1eyeg8+Wcd885b8g539DuXYDOlXN+7r/9tSnnfEm79wI6n886QKvknI/JOS/LOT+fc/63nPOkdu9Ea3W1ewE6xtyU0gerqnop5zwmpXR3zvmhqqoebPdiQEe7LKX0QLuXADpbVVVb//7/zzkPTSn9R0rp5vZtBBTEZx1gi8s5H5RSOj+ldHRK6f6U0pvauxHt4E1imqKqqp9WVfXS7//j7/7atY0rAR0u53xMSunplNL32r0LUJQZKaWnUkqL270I0Nl81gFa6OyU0jlVVd1XVVVPVVUrq6pa2e6laC0lMU2Tc7485/xCSunRlNKqlNIdbV4J6FA559ellM5JKf11u3cBinN8Sum6qqqqdi8CdC6fdYBWyTkPSClNSCl155yX55xX5JwvzTkPbvdutJaSmKapquqklNKwlNKklNKClNJL9f8NgD/auSmlr1RV9at2LwKUI+e8U0rpgJTSte3eBeh4PusArfLGlNLAtPl/LTUppbR3SulPU0p/286laD0lMU1VVdWmqqruTSntkFKa3e59gM6Tc947pXRgSunidu8CFOe4lNK9VVX9ot2LAJ3LZx2gxV783f+9pKqqVVVVrUkpzUspHdrGnWgDf3AdW0pX8nsSA1vGlJTSLimlJ3POKaW0dUppQM75LVVVjW/jXkDnOy6l9Ll2LwF0vCnJZx2gRaqqWp9zXpE2/9lSFCz77dTorZzzyJTS/0op3ZY2/xuoA9Pm327i2KqqvtHO3YDOk3MeklJ63X/6W6elzV9Iza6qanVblgI6Xs55v5TSd1JK21VV9Wy79wE6l886QKvlnM9JKR2SUjospfRySunWlNLdVVWd2dbFaClvEtMMVdr8W0tcmTb/FiZPpJROVRADW0JVVS+klF74/X/OOT+XUtrgiyZgCzs+pbRAQQxsaT7rAG1wbkppRErpsZTShpTS/04p/X1bN6LlvEkMAAAAAFAwf3AdAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQsK5WPizn7E/Jgw5QVVVu9w6NcOdAZ+gvd05K7h3oFP3l3nHnQGdw5wCt9Gp3jjeJAQAAAAAKpiQGAAAfwuuZAAAgAElEQVQAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAK1tXuBfqqHXbYIcx89KMf7fVzPvCBD4SZkSNHhpl58+bVzjdu3BiesW7dujDzla98pXb+zDPPhGf89re/DTMAnehrX/tamPmzP/uz2vmYMWPCM1atWtXwTgDAlrHHHnuEmcWLF9fOb7755vCMk08+ueGdADrJiBEjwszs2bNr51tvvXV4xic/+cmGd3o1jXRhU6dODTN33313r3d5Nd4kBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgILlqqpa97CcW/ewwFFHHVU7P/fcc8MzRo8e3ax1OsY3v/nNMHPXXXeFmcsuuyzMbNq0qaGdaL6qqnK7d2hEX7pz6HyDBg0KM4888kiY2XXXXWvne+yxR3jG8uXLw0x/0l/unJTcO9Ap+su9487p2+bNmxdmPvaxj/X6OV1dXb0+g/Zy51CimTNn1s7f/OY3h2ccc8wxYWavvfZqeKctqZEOa+rUqWHm7rvv7vUur3bneJMYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAoWFe7F2iXCRMm1M532WWX8IwHHnig13ssXbo0zOy4445hZtiwYbXzgQMHhmfsvffeYSbynve8pymZgw46KMxMmzatdr5p06bwDGDLiO6klFIaNGhQmFm9enUz1mmJ7bbbLszsuuuuLdgEANjStt566zAzadKkXj9n/vz5vT6jv2nk+3a//fYLM8cdd1zt/POf/3x4xo9//OMwA7zSJz7xiTDzuc99rnY+YMCAZq3Ta3fddVeY+dGPflQ7P++888Iznn/++YZ32hK8SQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABSsq90LtMvVV19dO//GN74RnnHfffc1a50tbuDAgWFm//33DzOjRo2qnX/hC18Izxg+fHiYOfTQQ8NMzjnMAO0xc+bMMHP22WeHmXe+852185UrVza805b2uc99rinnbNiwoXb+0ksvNeU5AMCrGzRoUO38uuuuC8/40z/9017v8e1vf7vXZ/Q1kydPrp1fddVV4Rm77757r/do5HPkj3/8414/B/qTGTNmhJk5c+aEmT333DPMDBgwoKGdemv+/Pm185NOOik84/nnnw8z0ddx/YE3iQEAAAAACqYkBgAAAAAomJIYAAAAAKBgSmIAAAAAgIIpiQEAAAAACqYkBgAAAAAomJIYAAAAAKBguaqq1j0s59Y9jLZYvHhxmNlvv/2a8qzx48fXzh9++OGmPIdXqqoqt3uHRrhztpzu7u7a+Y033hieceCBB4aZt7/97bXzH/3oR+EZzdLV1VU7f/zxx8Mzdt555zDz/ve/v3Z+3XXXhWd0mv5y56Tk3oFO0V/uHXfOlrPddtvVzlesWNGU58yfP792/t73vjc8o5Vf00e++tWvhpmZM2fWzgcPHtyUXT796U/Xzq+44orwjOeff74pu0TcObTKtGnTaufz5s0Lz2jka5pmWLZsWZg56KCDwsz69etr5xs2bGh4p07xaneON4kBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCdbV7AfqOAw88MMzMmTOndj5hwoRmrRMaNWpU7fzhhx9u0SZQnmOOOaZ23sh9cvvtt4eZhx56qOGdtrS3v/3ttfOdd945PGPt2rVhppHvFxg8eHCYGTt2bO18+vTp4RmNZP75n/+5dv7444+HZzz55JNh5rvf/W7tfI899gjPaCRz2223hZlmaGSX/fffv3a+2267hWcsX7684Z1ezWOPPRZmvv71r/f6OdAqU6ZMCTMXXHBB7Tzn3JRd1q1bVzuvqqopz2mGyZMnh5mZM2eGma233rp2vnHjxvCMK6+8Msxce+21tfPnn38+PAP6k9NPPz3MnHXWWbXzgQMHNmud0PXXX187b+Tbs2rVqmatQ/ImMQAAAABA0ZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAULCudi9AcwwcOLB2fs4554RnfOITnwgzXV29/yHz8ssvh5kxY8aEmV/+8pe93gV4pb333jvMXHTRRbXzjRs3hmdceumlYWbTpk1hplUmTpzY6zNOPfXUMLN27dpeP4f+7eCDDw4zF154YZgZN25cM9YJRb9mV1XVkj0akXMOM/1p376064ABA9q9AjSskXt2n332qZ038vPv+9//fpg544wzwkyrvPWtb62dL1iwIDxj8ODBYaanp6d2fvfdd4dnnHLKKWEGOsnpp58eZs4666wwE3VHjfiP//iPMPNXf/VXYWbx4sW18xdeeKHhnWgObxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwZTEAAAAAAAFUxIDAAAAABRMSQwAAAAAUDAlMQAAAABAwbravUDpBg0aFGbmzJkTZg488MDa+Tvf+c6Gd+qN+++/P8wcdthhYWbdunXNWAf4Ixx99NFhpqur/pePL33pS+EZixYtaninvuBjH/tY7fyll14Kzxg8eHCz1qGD/eM//mOY+fWvfx1m7rzzztr5woULwzNuueWWMNMMf/u3fxtmop+DzfLzn/+8dr58+fLwjLvuuqspu9x7772188ceeyw8Y+uttw4zDz30UO389a9/fXgG9BUf+tCHwswpp5zS6+c899xzYeZTn/pUmFm/fn2vd2lEI3fBZz7zmdr58OHDm7LLl7/85dr52Wef3ZTnQH8ybdq02vlZZ50VnjFw4MBe7/GDH/wgzCxevDjM9Lev9djMm8QAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDButq9QKcbOnRo7fyaa64Jz5g+fXqTttny7r///jCz3XbbhZnf/OY3YWbTpk0N7QT8P6NGjQozJ598cq+fM3/+/F6f0Uqf+tSnwkz0fXfxxReHZ8yaNSvM3H333bXz5cuXh2fQv02ZMiXMPPLII1t+kRZ6+eWXe33GY489FmbOP//8MHPTTTfVzjds2NDwTn3B/vvvH2Ze//rXt2ATaI7tt9++dn7GGWeEZ7zmNa8JM9HP9eOPPz48Y8mSJWGmVf7u7/4uzEybNq3Xz1m6dGmYOfPMM2vnq1ev7vUe0JfstNNOYebss8+unQ8cOLBZ69Q67bTTwsyDDz7Ygk1oB28SAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMGUxAAAAAAABVMSAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMGUxAAAAAAABetq9wKdbtKkSbXz6dOnt2iT1jj55JObkvnOd74TZr73ve/Vzm+44YbwjFWrVoUZ6CSnnHJKmBk6dGiYWbhwYe38u9/9bsM79QWHH354mPntb39bO3/iiSfCMxq5/wYOHBhm6GyPPPJIu1doub/5m78JM1dffXXt/LHHHmvWOv3GrrvuGmb+4R/+IczknGvnc+fObXgn6I2RI0eGmW9+85u18x133LEpu0Rfa3zjG99oynOa4dBDDw0zJ554Ygs2aewz1erVq1uwCfQdjXwNsOeee7Zgk5QeffTR2vn2228fntHT0xNmHnrooYZ3ou/wJjEAAAAAQMGUxAAAAAAABVMSAwAAAAAUTEkMAAAAAFAwJTEAAAAAQMGUxAAAAAAABVMSAwAAAAAULFdV1bqH5dy6h/URb3jDG2rnF1xwQXjGvvvuG2aWL1/e8E69sc0229TOJ06c2JI9GvHwww+HmY9//ONh5p577mnGOh2lqqrc7h0aUeKds99++9XO77777vCMn//852EmupfWr18fntEq++yzT5i59957w8yPf/zj2nl3d3d4xlZbxf9u9k/+5E/CTGn6y52TUpn3Du1z3XXXhZn3ve99YWbZsmW18wkTJoRnbNiwIcz0J/3l3ulPd852220XZr773e+GmbFjx/Z6lxdeeCHMDBs2rNfPaYYhQ4aEmeeffz7M9PT0NGOdpog+DzVr1+iOvOOOO8Izbr755qbsEnHn9F+33XZbmDnooIPCTFdXVzPWaYmnn346zNx6661h5hOf+ETtvC99TdlpXu3O8SYxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQsK52L9Dp1q9fXzs/4YQTwjO23XbbMLN27dqGd+qNwYMH18532WWX8Iw999wzzJx22mlhZsKECbXzvfbaKzzjW9/6Vpi54IILaueXXnppeMaaNWvCDES6u7vDzNe+9rXaeVdXfO0//fTTYWbUqFG9fs7q1avDTDP89V//dZh57WtfG2be8Y531M5XrVoVnnHssceGGYDfO/jgg2vnRxxxRHhGI3fT0UcfXTvfsGFDeAZE5s6dG2bGjBkTZqqq6vUuN998c5jZfvvta+cvvvhieMYb3vCGMDNs2LDa+Re/+MXwjJ6enjDTjO+3Zon2bdau0We3jRs3hmc08mOFsk2aNCnMNPK1UX8yfPjwMHPccceFmXHjxtXOL7nkkvCM22+/PcysW7cuzLCZN4kBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICCKYkBAAAAAAqmJAYAAAAAKJiSGAAAAACgYEpiAAAAAICC5aqqWvewnFv3MPq1IUOGhJkjjjiidn7DDTc0a51a119/fZh5//vfv+UXaaGqqnK7d2hEp905u+++e5j52c9+1oJNUnr55Zdr5xs3bgzP+O1vfxtmom9PzvEPxQkTJoSZRs65//77a+fvfe97wzN+8YtfhBleqb/cOSl13r3DljN8+PAwc88999TO99xzz/CMs846K8z8/d//fZgpTX+5d/rTnbNp06Yw08qvTSNPPvlk7fzZZ58Nz3jTm94UZrbZZpuGd3o1jXyO6Uvft9G99MADDzTlOT/84Q9r588991xTntMM7py+661vfWvt/F/+5V/CMxrpN9atW1c7v/3228MzFixYEGaa4dRTTw0zBxxwQAs2SemUU04JM5deemkLNulfXu3O8SYxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQMCUxAAAAAEDBlMQAAAAAAAVTEgMAAAAAFExJDAAAAABQsFxVVeselnPrHkbxtt122zDz7W9/O8yMHz++dt7Iz6GPfOQjYebqq68OM31FVVW53Ts0otPunE9+8pNh5vzzz6+dr169utdnpJTSPvvsE2Za4c///M/DzLBhw8LM/fffH2b222+/2nlPT094Bn+c/nLnpNR59w5/nOHDh4eZb37zm2Fm4sSJtfNG7q5DDz00zKxbty7MlKa/3Dv96c7ZtGlTmGnl16aRnOt/CPSnXVNqbN9Vq1bVzr/3ve+FZ1x55ZVh5r777gszpXHn9F0bNmyonQ8cOLApz3n44Ydr5+9+97vDM9avX9+UXSIjR44MM7NmzQozn/nMZ2rnQ4cODc949tlnw8wNN9xQOz/55JPDMzrNq9053iQGAAAAACiYkhgAAAAAoGBKYgAAAACAgimJAQAAAAAKpiQGAAAAACiYkhgAAAAAoGBKYgAAAACAgnW1ewHYUtauXRtmTjjhhDDz4IMP1s5zzuEZV111VZi55ZZbaufr168Pz6CzLVq0KMxMnjy5dv6DH/wgPGPevHkN77Sl7bbbbrXzww8/vCnPmTNnTpjp6elpyrOAznfxxReHmYkTJ4aZ6DPGueeeG56xbt26MAOtcMUVV4SZj3zkIy3YpEx33HFHmLn11ltr51dffXWz1oE+4V3veleY2Wqr1rxbefrpp9fO+1If8NRTT4WZiy66KMzsueeetfMZM2aEZwwbNizMDB48uHa+9dZbh2c899xzYaYTeJMYAAAAAKBgSmIA+L/s3X203uOZN/zzlK2IVI2EoJJQ78V462QhaYsySoZVLzHVKaVDjfeI6mg1ZYaJJSoJpUHHS9ToQxS3UtrVepmoaDtTYzQVYTxNomoqIk2TSOTl9/wRfdY99y2/46rrynXtfZ2fz1pZXXIcPc5j79hnrv3dP3sDAABAwYTEAAAAAAAFExIDAAAAABRMSAwAAAAAUDAhMQAAAABAwYTEAAAAAAAFExIDAAAAABQsV1XVvsNybt9hXeQb3/hG2DNw4MDa+uzZs8MZl156aaMr9QrrrVf/NY6RI0eGMy666KKw57DDDmt4p7V59NFHw54jjjiitr5ixYqm92iVqqpyp3dohDun7/vKV75SW7/88svDGQsWLAh7tttuu7DnD3/4Q9jDutFX7pyU3DvdYKONNqqtT5gwIZxx+umnhz39+vULe2666aba+rnnnhvO6E2vH/qSvnLvuHPWnf79+9fWn3jiiXDGPvvs0/QeS5cuDXsOPvjgsOfnP/9507uw7rhzeq9ly5bV1tdff/2WnPP3f//3tfVJkyaFM1atWtWSXXqL+++/P+w58sgjmz7n1FNPDXtuvfXWps/pTdZ253iSGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKFhPpxco3UEHHRT2fP7znw97Ntxww9r65MmTG96pGYMGDQp7/vzP/zzsOeuss8KeTTbZpLZ+8MEHhzNa4cknnwx7xowZE/asWLGiFetAV2nkjoxccMEFYc8f/vCHps8BusM555xTWz/zzDNbcs7NN98c9pxxxhktOQv40/X01H+qvOmmm4YzqqoKe37yk5/U1s8///xwxi9+8YuwB+jdrrzyytr6nnvuGc54+umnw57bb7+9tt6qz4uivCallLbbbrvaeiPZUSPmzp1bW58xY0ZLzukGniQGAAAAACiYkBgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAguWqqtp3WM7tO6yPGDlyZNjz8MMPhz39+/evra9YsSKc8eyzz4Y9kU033TTs2WGHHZo+p52efPLJ2vrFF1/c9Iy+pqqq3OkdGuHO6d0OPfTQsOeBBx6ora9cuTKcsf3224c9v/vd78IeOqev3DkpuXd6u7PPPjvsmThxYm29X79+4YwpU6aEPWPGjAl7GrnjWDf6yr3jzll3ttxyy9r6K6+80pJz7rjjjtr6ySef3JJz6N3cOb1X9NphxIgR4YzRo0eHPTm351+Bl19+ubb+4IMPtuScY445JuzZZpttmj5n6dKlYc/nP//52vq0adOa3qOvWdud40liAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYLmqqvYdlnP7DusiV1xxRdgzduzY2npPT0+r1uk1Vq5cWVu/7bbbwhkPPPBA2PPoo4/W1t96661wRrepqip3eodGuHN6t3PPPTfsmTx5cm393/7t38IZw4cPb3gneqe+cuek5N7ppJ122insefzxx8OewYMH19bnzJkTzthjjz3CniVLloQ9dE5fuXfcOevOlltuWVt/5ZVXWnLOwoULa+vz5s0LZ+y9994t2YXOced0t7PPPjvsOeCAA2rrxx57bDijXbnPCy+8EPbceuutYc8hhxxSW4+ymJRSuv/++8OeRvYtzdruHE8SAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABevp9ALEvvzlL4c9Tz75ZG39L//yL8MZZ599dtjz3e9+t7Y+aNCgcMZPf/rTsOf6668PexYvXlxbX7hwYTgD6JxXX3017Mk519ZvvPHGVq0D9HEf/ehHw56tttoq7Fm0aFFt/cADDwxnLFmyJOwBerfoc43bb789nHHSSSeFPdHrofHjx4czgN7tuuuua7pnjz32CGccddRRDe+0NjvssEPYU1VV2NNIHnPKKafU1gcMGBDOmD17dthD4zxJDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAXLVVW177Cc23cYsM5UVZU7vUMj3DnQHfrKnZOSe2dd6tevX239+9//fjjj0EMPDXsmT55cWx87dmw4g76vr9w77hzoDu4coJ3Wdud4khgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAAChYrqqqfYfl3L7DgHWmqqrc6R0a4c6B7tBX7pyU3Dvr0i677FJbnzlzZjhj1qxZYc/w4cNr60uWLAln0Pf1lXvHnQPdwZ0DtNPa7hxPEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAXr6fQCAAAQ+d3vfldbv+iii8IZjz/+eNizZMmSRlcCAICu4UliAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKFiuqqp9h+XcvsOAdaaqqtzpHRrhzoHu0FfunJTcO9At+sq9486B7uDOAdppbXeOJ4kBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICC5aqqOr0DAAAAAAAd4kliAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYlpiZzzrjnnR3POv885v5RzPrrTOwHdK+e8bc75+znnN3POr+Wcr8s593R6L6A75Zw3yDnfnHOek3P+Q875mZzz4Z3eC+h+Oecdc87Lcs53dHoXoPu5c8omJKZp7wQz/yul9GBKabOU0hdSSnfknHfq6GJAN/tmSul3KaWtUkp7pZQ+nlI6s6MbAd2sJ6U0L625az6QUhqXUro757xtB3cCynB9SunnnV4CKIY7p2BCYlphl5TS1imlSVVVraqq6tGU0k9SSid2di2gi22XUrq7qqplVVW9llJ6JKW0W4d3ArpUVVVLqqq6tKqqX1dVtbqqqgdTSv9vSmnfTu8GdK+c86dTSgtTSj/u9C5A93PnICSmFfJafm/3di8CFOOalNKnc879c84fTCkdntYExQDrXM55cEppp5TSzE7vAnSnnPMmKaV/TCld0OldgO7nziElITGtMSut+c++L8w5r59z/su05j/H7N/ZtYAu9kRa8+TwopTSKymlf0sp3d/RjYAi5JzXTyn9S0ppalVVszq9D9C1Lksp3VxV1bxOLwIUwZ2DkJjmVVW1IqX0qZTSqJTSa2nNV57uTmuCG4CWyjmvl1L6QUrp3pTSximlQSmlP0spXdnJvYDu98798+2U0tsppbM7vA7QpXLOe6WUDkkpTer0LkD3c+fwR34SPC1RVdV/pjVPD6eUUso5P5VSmtq5jYAutllKaUhK6bqqqpanlJbnnG9NKV2eUvpSRzcDulbOOaeUbk4pDU4pHfHOF8kB1oUDU0rbppTmrrl60oCUUr+c84erqtqng3sB3enA5M4hpZSrqur0DnSBnPOfp5RmpzVPp5+ZUjorpbTLOwEOQEvlnF9OKd2UUvp6WvMi5taU0tKqqv6mo4sBXSvnfENKaa+U0iFVVS3u9D5A98o5908pbfK//dYX05oA54yqql7vyFJA13Ln8Ee+3QStcmJK6bdpzfcm/kRK6VABMbAOHZNS+mRK6fWU0ksppZUppfM7uhHQtXLOw1JKp6c1IfFrOefF7/zyhSmg5aqqWlpV1Wt//JVSWpxSWiasAdYFdw5/5EliAAAAAICCeZIYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAoWE87D8s5+yl50AWqqsqd3qER7hzoDn3lzknJvQPdoq/cO+4c6A7uHKCd1nbneJIYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAJv2mjEAACAASURBVAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAoWE+nFwAAgL5iww03bMmcZcuWtWQOAAC0gieJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAgvV0egEA6AvWW6/+66p/93d/F874q7/6q7DnkEMOqa1fdtll4Yyrrroq7Fm2bFnYA91mxIgRYc+FF15YW99yyy3DGVVVhT2vvfZa2HPLLbfU1p944olwxqJFi8IeoPuNHj067Ln77rtr69OmTQtnHH/88Q3vBEDv4kliAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKFiuqqp9h+XcvsOAdaaqqtzpHRrhzqGVLr300tr6uHHj2rNIA+68886w57TTTqutL1u2rFXrNK2v3DkpuXfWpQ033LC2fskll4QzxowZE/ZssMEGDe/Uaddcc03Yc/7557dhk+7TV+4ddw6NGj16dNhz9913N31Ozn3iQ6fXcecA7bS2O8eTxAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMF6Or0A7bHjjjuGPaNGjQp7jj322Nr6r3/963DGzTffHPY8/vjjYQ9Aq1xyySVhz1e/+tWmz3nggQfCnjfffLO2/rnPfS6c8ZnPfCbsmT17dm39sssuC2dAq+y9995hz5QpU2rrw4cPb8kuzz77bG39hhtuaMk5I0aMCHs++9nP1tY/+clPhjO+9KUvhT0rVqwIe4C+bb/99uv0CgAt18hroalTp9bWt9hii3DGvvvuG/b84he/CHt6O08SAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABevp9AK0xumnn15bP/nkk8MZw4cPb3qPESNGhD3HH3982DNmzJiwZ8qUKQ3tBPRdgwYNqq2vt178tc4DDjgg7Bk3blzYs3Tp0tr6ddddF8746le/Gvb09NT/1fyRj3wknLHbbruFPZtttlnYA62w9957hz3jx48Pe6LXKT/4wQ/CGRMmTAh7nnzyydr6ihUrwhmNuO+++8Kez372s7X1nXfeOZwR3Skpte5tAnqvIUOGND1j3rx5LdgEeDcXX3xxbX3kyJHhjBNPPDHsmT9/fsM79QUPPfRQ2FNVVW398ssvD2c8//zzDe/Ul3mSGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAArW0+kFiN14441hzymnnFJb79evX0t2efXVV2vrDz30UDhj9OjRYc+kSZMa3mltpkyZ0vQMYN0ZNWpU2POd73yntj5gwIBwRlVVDe9U5+Mf/3ht/ZlnnmnJOeuvv35tfbfddmvJOdAup59+ethz2GGHhT2PPfZYbf2YY44JZ7z11lthT1+ycuXKTq8AfdLYsWPDnhkzZjRV72222WabTq8AxRo2bFjYc+6559bWf/jDH7ZqnV5j3333ra1PnDgxnLF06dKw54orrqitjx8/PpxRCk8SAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABevp9AKlmzRpUthz2mmnhT1VVdXW//u//zuccfnll4c9d955Z2194cKF4YwHHngg7Pne974X9lx55ZW19R//+MfhjNmzZ4c9wJ9u++23D3vuuOOOsGfjjTeurTfyMbzjjjuGPY3cS88++2zY0wo777xzW86BdnnooYfCnoEDB4Y9J510Um39rbfeanin3uCEE05oesbUqVPDnr72foF2uPrqq5uekXNuwSbts//++zc94+mnn27BJlCeiRMnhj3Ra6Hp06eHM+bPn9/wTr1BlHWNGDEinPHMM8+EPePHj294p9J5khgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAAChYT6cXKN1OO+3UlnMuuuiisOf2229vwyYpvfHGGy2Zs/HGG9fW//mf/zmc8bGPfawlu0Bp+vfvX1u/5ZZbwhmbbLJJ2DN8+PDa+qxZs8IZ48ePD3see+yxsGf16tVhT2SvvfYKex5++OGmz5k5c2bYc9VVVzV9DjTie9/7Xkt6+pJPfvKTYU8jH4Pz58+vrf/7v/97wztBKYYMGdJrzpk3b14bNmmfbnt7oF2iv89TSinnXFs/+uijwxk33XRTwzv1BlEeE71PUmrsfUvjPEkMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABevp9ALdbuDAgbX1ww8/PJyRcw57Xnrppdr6vffeG87oTRp5myMjR45swSbAuxkzZkxtvVUffwsWLKitL1myJJxx3nnntWSXfv361dYPPvjgcMa3v/3tsGfQoEG19eXLl4czTj311LDn1VdfDXuA/9sHP/jBsOfaa68Ne3p64pfh0Z1xww03hDOgNNFrlEbNmzevqXo77b///m0555577mnLOdCX7LrrrmHPpz71qbCnqqra+vjx4xveqTe4+OKLw56dd965tv7888+HM0488cSGdyLmSWIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgPZ1eoHRVVbVkzvz582vrixcvbsk57dKq9wtAI4YNGxb23HLLLbX1Aw88sCW7LFu2rLZ+0EEHhTN+9rOftWQXKFG/fv1q61/84hfDGTvssEPY8+KLL4Y91157bdgD/E9jx45tyZzJkye3ZE47HHfccW05Z8aMGW05B3qT/v3719bvueeecMbmm28e9vzTP/1TbX369OnhjHZp5HOnc889N+zJOdfW77vvvnBGlIXxp/EkMQAAAABAwYTEAAAAAAAFExIDAAAAABRMSAwAAAAAUDAhMQAAAABAwYTEAAAAAAAFExIDAAAAABRMSAwAAAAAULCeTi/Q7VatWlVbX7JkSThjwIABrVqnVzjyyCM7vQJQkLPOOivsueSSS8KegQMHNr3L7Nmzw56jjz66tj5r1qym94BSbbvttmHPhRdeWFs/88wzwxlVVYU9Rx11VNgzZ86csAdKc/XVV7flnBkzZrTlnFYYO3ZsS+ZMmzatJXOgm0R/pzfyd34jPX3JoEGDwp5GPnfqtvdLN/AkMQAAAABAwYTEAAAAAAAFExIDAAAAABRMSAwAAAAAUDAhMQAAAABAwYTEAAAAAAAFExIDAAAAABSsp9MLdLuFCxfW1qdPnx7OOPzww8Oenp76P8r11ou/HrB69eqwJ7LTTjuFPX/zN3/T9Dmtct5554U911xzTRs2gb4l59xUvVHjxo2rre+9997hjL322ivsqaoq7HnzzTdr65MnTw5nXHbZZWEP8N585jOfCXv+4R/+IezZfvvta+uLFi0KZ1x88cVhzwsvvBD2QGn233//sGfs2LFt2CTeZZtttglnTJs2rek92vX2ppTSpEmT2nLO6NGja+tDhgwJZ1x99dVhT6tej1K2j3/847X1XXfdNZzRyL+L0Zx99903nNEKRx99dNhzzDHHhD2NvM1RTyPv20beL/Pnz6+tz5kzJ5xRCk8SAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABctVVbXvsJzbd1gfcdxxx4U9d999d9gT/TlOnTo1nDF48OCwJ7LnnnuGPVtttVXT57TKI488EvaMGjWqDZv0LVVV5U7v0Ah3zroT3UvHHntsW/ZYb734a52rV68Oe2655ZawZ8KECbX1F198MZzBe9NX7pyU3Dvr0jnnnFNbv+aaa1pyzje+8Y3a+sSJE8MZc+bMackudE5fuXe67c6ZO3du2DNkyJA2bEI3mDdvXm196NChbdok5s7pvW644Yba+qmnnhrOyDn+440ynVbMaGROK2b0tl2mT59eWz/wwAPDGd1mbXeOJ4kBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICC9XR6gdLdc889Yc9TTz0V9hxwwAG19ZNPPrnRlZqScw57qqoKex588MGwZ9ddd62tb7/99uGMI444IuzZbLPNausLFiwIZ0BvscUWW4Q9X/ziF8OekSNHtmKdps2YMSPseeSRR8KeCRMmhD3Lli1raCfgfxoxYkTYc9FFF4U9hx12WNO7TJ48Oey58MILa+urVq1qeg8o0ejRo8OeIUOGtGETusG0adNa0gORO++8s7Y+bNiwcMagQYOa3mPfffcNe+bPnx/2zJ07t7beyK6NvM2NiHa54oorwhmNvM333ntvwzuVzpPEAAAAAAAFExIDAAAAABRMSAwAAAAAUDAhMQAAAABAwYTEAAAAAAAFExIDAAAAABRMSAwAAAAAUDAhMQAAAABAwXJVVe07LOf2HdZFdtppp7Dnpptuqq1/6EMfCmcMHjw47Fm8eHFt/bnnngtn3HjjjWHPXXfdFfb85Cc/qa0PHz48nJFzDnsGDRpUW1+wYEE4o9tUVRW/43qBvnTnbLrppmHPPvvsE/Z897vfra1vsMEG4YxGenqLRnZduXJlGzZhXeord05KfeveaZVRo0bV1u+7775wRk9PT9jz5ptv1tYvvPDCcMZtt90W9qxevTrsKc3GG28c9ixZsqQNm7RPX7l3+tKdM2TIkLBn7ty5Yc+0adNq6xdccEE4Y968eWFPtO/o0aPDGfvtt1/Y08icVpgxY0bY88orrzR9ztNPPx32RO//6M+4G7lziDTyueD8+fPDnuie/cIXvhDOmDJlStjTSNb4ta99rbY+fvz4cAbvzdruHE8SAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMFyVVXtOyzn9h3G/7DZZpuFPR/4wAfCnuXLl9fWFy1aFM5YvHhx2NOIGTNm1NaHDx8ezsg5hz2DBg2qrS9YsCCc0W2qqorfcb1Ab7pztt1229r67bffHs4YMWJE03u89NJLYc8bb7wR9qy3Xv3XGP/iL/6i4Z3qvPbaa7X1D37wgy05h96tr9w5KfWueyey0UYbhT1XXHFF2HPaaac1fc5Pf/rTsOfUU0+trc+cOTOc0YjtttuuqXqjnnjiibBn1apVtfWtt946nLHLLruEPc8991xt/eabbw5nvPnmm2HPBRdcUFufP39+OKNd+sq905fuHN5duz4fb+TzHjrHnUO7DBs2rLb+s5/9LJyx+eabhz2vv/562DN48OCwh3VjbXeOJ4kBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICC9XR6AdpjwYIFLelpl0033TTsGTx4cNPnPPXUU2HP73//+6bPgSOOOKK2PmLEiJacM3bs2Nr6HXfcEc544403wp5//dd/bXinZvzHf/xHW86BEjVyHxx99NFt2CSlD3/4w2HPXXfd1YZNUtp8882bqjfqhRdeCHtWrVpVW2/k9dLWW28d9sybN6+2vsUWW4QzNthgg7Bnu+22q60fddRR4YyFCxeGPdBb7L///m05Z+LEiW05B+h+VVW1pOfee+9txTq0mSeJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAgvV0egF4N5tvvnnYM2zYsKbP2XjjjcOefv361dZXrVrV9B50vyOPPLLpGXfddVfYc+2119bWc87hjH322Sfseeutt8KeyIsvvhj23HrrrU2fA7y72bNnd3qF/9/73//+sOfDH/5wGzZJafXq1bX1X//61+GMrbfeOuzZeeedG11prV599dWwp5F9I4sXLw57Nthgg7Bn5MiRtfXbbrstnPGpT30q7IHe4vzzz2/LOZMnT27LOUDfN2fOnNr6vHnzwhlbbLFFq9ahl/EkMQAAAABAwYTEAAAAAAAFExIDAAAAABRMSAwAAAAAUDAhMQAAAABAwYTEAAAAAAAFExIDAAAAABSsp9MLwHuVc256xm9/+9uw5+233276HJg8eXJtff/99w9n/PVf/3XYs3z58tr6nDlzwhnjxo0Le1qhkY+tH/7wh23YBMp0+eWXhz0DBw4Me4YOHdr0Lvfdd1/Y88orrzR9TiNWrlxZW//BD34Qzhg5cmTY84EPfKDhndZm+vTpYc+iRYuaPmfYsGFhz+677970OY899ljTM6A3GT16dNMzZsyYEfbMmzev6XOAMlx88cW19V122SWcUVVV2NPIazt6H08SAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABevp9ALwbgYOHBj2VFXV9Dkvv/xy0zOgETNmzKitP/744+GMI488Muw56aSTGl1pnXr00UfDngsuuCDsWbRoUSvWAd7FkiVLwp4vfOELbdik+zz55JOdXqGl5syZ05Ie6CZjx45tyzn33HNPW84ByrDRRhvV1vv37x/OeOONN8KeWbNmNbwTvYcniQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIL1dHoBeDdbb711W86ZM2dOW86BRYsW1dY//elPhzM+8YlPhD1f+9rXausf+chHwhmN+Na3vlVbv+2228IZ//mf/9mSXQAA2m2//fZryznTpk1ryzlAGe67777a+kUXXRTOGDhwYNgzaNCgsGfu3LlhD+3lSWIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAoWE+nF4B389xzz7XlnGHDhrXlHIgsW7Ys7HnooYda0gMAQHOOP/74Tq8A8Cf72Mc+VlvPOYcz5s6d25Ieeh9PEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAXr6fQC8G5+85vfhD1f/vKXa+tXXHFFOOPxxx9vdCUAAACAPutXv/pVbb2qqnDG+eefH/bMnz+/4Z3oPTxJDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFCxXVdW+w3Ju32HAOlNVVe70Do1w50B36Ct3TkruHegWfeXecedAd3DnAO20tjvHk8QAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBclVVnd4BAAAAAIAO8SQxAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMS0RM75jpzzb3POi3LOs3POp3Z6J6D75Zx3zDkvyznf0eldgO6Xc/50zvn5nPOSnPN/5Zw/2umdgO6Uc94s53zfO/fNnJzzZzq9E9C93DmklFJPpxega1yRUvrbqqqW55x3SSk9nnN+pqqqf+/0YkBXuz6l9PNOLwF0v5zzoSmlK1NKf51S+llKaavObgR0uetTSm+nlAanlPZKKT2Uc362qqqZnV0L6FLuHDxJTGtUVTWzqqrlf/zHd35t38GVgC6Xc/50SmlhSunHnd4FKMI/pJT+saqqp6uqWl1V1W+qqvpNp5cCuk/OeeOU0rEppXFVVS2uqurJlNIDKaUTO7sZ0I3cOfyRkJiWyTl/M+e8NKU0K6X025TS9zu8EtClcs6bpJT+MaV0Qad3AbpfzrlfSukjKaXNc84v5ZxfyTlfl3PeqNO7AV1pp5TSqqqqZv9vv/dsSmm3Du0DdDd3DiklITEtVFXVmSml96eUPppSujeltLz+/wHwnl2WUrq5qqp5nV4EKMLglNL6KaXj0prXOXullPZOKX21k0sBXWtASun3/8fv/T6t+VwLoNXcOaSUhMS0WFVVq975TxO2SSmd0el9gO6Tc94rpXRISmlSp3cBivHWO//7jaqqfltV1fyU0sSU0hEd3AnoXotTSpv8H7+3SUrpDx3YBeh+7hxSSn5wHetOT/I9iYF148CU0rYppbk555TWfOW7X875w1VV7dPBvYAuVVXVmznnV9Kan7kAsK7NTin15Jx3rKrqxXd+b8+Ukh8gBawL7hxSSinlqvJal+bknLdIKR2cUnowrXnS5pC05ttNfKaqqv/Vyd2A7pNz7p/+51e6v5jWhMZnVFX1ekeWArpezvkfU0qHp5RGpZRWpDU/0OXxqqrGdXQxoCvlnP+ftOYLU6emNd/i5vsppQOqqhLaAC3nziElTxLTGlVa860lbkhrvoXJnJTSGAExsC5UVbU0pbT0j/+cc16cUlomIAbWsctSSoPSmqdtlqWU7k4p/VNHNwK62ZkppVtSSr9LKb2R1nwxXFgDrCvuHDxJDAAAAABQMj+4DgAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKFhPOw/LOfspedAFqqrKnd6hEe4c6A595c5Jyb0D3aKv3DvuHOgO7hygndZ253iSGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKFhPpxcAAADWjcMOOyzsueyyy2rrs2bNCmf87d/+bdizYsWKsAcAgM7wJDEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFAwITEAAAAAQMGExAAAAAAABRMSAwAAAAAUTEgMAAAAAFCwnk4vAAClOO6448KeadOm1dZ/9atfhTN22223hncC+q4JEyaEPeedd17Ys/7669fWc87hjEZ6AID3pqcnju9Gjx4d9gwdOrS2PmrUqHDGRz/60bCnqqqwZ86cObX1Sy+9NJwxderUsIfGeZIYAAAAAKBgQmIAAAAAgIIJiQEAAAAACiYkBgAAAAAomJAYAAAAAKBgQmIAAAAAgIIJiQEAAAAACtbT6QX6smHDhoU9F154YdPnbLbZZmHPbrvtVlvffffdm96jETnnsKeqqrBn2bJlYc9XvvKV2vo111wTzgBop3333TfsWb16dW29kTsU6A4nnXRSbX3MmDHhjEbujEsuuaS2fuWVV4Yz3n777bAHAHhvzjrrrLBn4sSJbdiksdcWjfQMHTq0tn7DDTc0vFOdqVOntmROCTxJDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFCxXVdW+w3Ju32FtcMkll4Q948aNa/qcnHPY084/xzrt3PWJJ56orX/iE59oyTn836qqiv+ge4Fuu3Na4bDDDgt7Jk6c2PQ5e+65Z9izcuXKps/pTYYNGxb2zJgxI+wZPHhwbf35558PZ+y+++5hT1/SV+6clNw7NO5zn/tc2POtb32rtt7T0xPOaOT10GOPPRb2lKav3DvunN5t6623Dnt22GGHNmyS0vTp08Oe3vI5ZYncOZxzzjm19csvvzycMWDAgKb3mDNnTtgzZsyYsGeDDTYIe8aOHVtbHz58eDhj7ty5Yc92220X9pRmbXeOJ4kBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICCCYkBAAAAAAomJAYAAAAAKJiQGAAAAACgYEJiAAAAAICC9XR6gb5s6dKlYc/y5ctr66tXrw5n9O/fP+xZuHBhbf3tt98OZ8ycOTPs2W233Wrr73vf+8IZm266adgDvDfXXHNNbf2EE04IZwwcOLDpPXLOTc/oaw4//PCwZ/DgwW3YBOi0vffeO+y56aabwp6envqX6k899VQ445e//GXYA/zpNtlkk7Bn6tSpYc/BBx/cinVCZ599dtgzZcqUNmyS0pe+9KXa+hZbbBHOaORz8a997WsN7wSdduqpp9bWBwwY0JJzXnzxxdr6EUccEc54+eWXW7LLO1yRLwAAIABJREFU/fffX1v/+te/Hs5o5G4788wza+vf/OY3wxml8CQxAAAAAEDBhMQAAAAAAAUTEgMAAAAAFExIDAAAAABQMCExAAAAAEDBhMQAAAAAAAUTEgMAAAAAFKyn0wv0ZVdddVXYM3369Nr68uXLwxnbb7992PPEE0/U1l9//fVwRisceuihYc/DDz/ckrN+85vftGQOdJM999yztj5w4MBwxqpVq8KeG2+8sba+cuXKcAbvzbe//e1OrwAEvv71r4c966+/fthz/fXX19bPOeechneCUmy11VZhzz777BP2TJ48ubbe0xN/Kj106NCwp10mTJgQ9gwYMKC2Pnr06HDGn/3Zn4U922yzTW39fe97XzijkderJ5xwQm39jDPOCGf86Ec/CnsgMmrUqLBnjz32qK1XVdWSXW6++eba+ssvv9yScxqxYsWK2vott9wSzmjkXorugm9+85vhjFJ4khgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAACiYkBgAAAAAoGBCYgAAAACAggmJAQAAAAAKJiQGAAAAAChYT6cX6HZPP/100zOeeeaZFmzSGnvssUdt/Tvf+U5LznnrrbfCnkmTJrXkLOgrenriK7uRnsiCBQvCnnPOOafpc7rN0Ucf3ZZzfvnLX7blHODdjRo1Kuw56KCDwp4f/ehHYc95553X0E5QkqFDh9bW/+Vf/iWcccABB7RqnXVu5syZLZmz0UYbhT3vf//7a+uPPPJIS3ZphRNPPDHs+dCHPlRb79+/f6vWoWCNfGyNHz8+7Mk519b/67/+K5xxxhlnhD2NvP7oLZ599tmwZ+rUqWHPKaecUlvfc889W7JLN/AkMfD/tXfvUVvWZb7Af3e8CL6ekMBGMMQTmkE6HqfMw5gK5WFoD6kheVyGaTJj7nBkSBQVl8fMQylaHpLASRCcbDqIuSd3OYp5mAblYBtN1AFTA17EIO79h+1pzTbv68nn4Xl4n9/ns1ZrzZrru677CvP25etdAgAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkrKPVB7Dx2H///cPM/fffXznv27dvuKMsyzBz1VVXhZknnngizEA7GT58eJj56Ec/WvdzJk2aVPeOHP3VX/1VQ/YsWbKkcr5w4cKGPAf40/r37185v/3228MdTz/9dJgZPXp0mFm/fn2YgXay9dZbh5mZM2dWzvfaa69GnVPp97//fZh55ZVXwsxFF11UOf/JT35S801VevToEWYWLVrUkGc1w8EHHxxmBg0a1IRLyN2QIUPCzNChQ8NM1JNMmzYt3PHAAw+EmXZTy3trm222qZwff/zx4Y6nnnqq5pu6M18SAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMY6Wn0AzbHPPvuEmTlz5oSZPn36VM7Lsgx3vPDCC2Hm7rvvDjMAzTRhwoTK+eabb96Q50TvyEWLFjXkOZCj970v/j5i+vTplfP3v//94Y7TTz89zLz66qthBtrJDjvsEGbuuuuuMLPXXns14py63XbbbWFm7NixTbik/ey5555hZtttt637OSNGjAgz9913X93Pob3deuutDdnzwx/+sHJ+5ZVXNuQ57WbhwoWtPqGt+JIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADIWEerD6AxTj755Mr5FVdcEe7o27dvg66pdtddd4WZ1157Lcz06dOncv7GG2/UfBNApEePHk15zqWXXtqU50COJk6cGGYOPfTQyvn06dPDHbNnz675JsjFrFmzwsxHPvKRJlyS0pIlS8LMGWecUTl/5JFHGnRNfoqiqJwPHz483LHTTjvVfccpp5wSZs4888y6n0N723777Ruy57vf/W7lvKurqyHPaTfz5s2re8cJJ5wQZs4///y6n9Md+JIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADIWEerDyDW2dkZZsaNG1c579u3b6POqdv555/fkMyyZcsq5zfddFO4Y/LkyWEGaH89e/YMM2eccUYTLklpxYoVTXkOtJtPfepTYWbixIlhpiiKyvny5cvDHX/5l38ZZp544okwA+1kzz33DDPr169vwiUp/cu//EuY+fGPf9yES/J08sknV86nTJnSlDuOO+64pjyH7u3AAw+snG+55ZbhjoULF4aZ733vezXfxB994hOfqHtHr169GnBJe/AlMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGSso9UHENtqq63CTL9+/ZpwycZlm222qZxfcMEF4Y7DDjsszHz2s5+tnC9dujTcAWzciqIIM3/xF3/RhEuAd9OnT5/K+XXXXRfu6OiIf/Qty7JyPm7cuHDHF77whTDz+c9/PszccccdYQb4811zzTWtPiFr55xzTqtPgJr17du3cr7JJpuEO2bNmhVmli1bVvNN/FEtv0eLfq9Xy+8Fc+FLYgAAAACAjCmJAQAAAAAypiQGAAAAAMiYkhgAAAAAIGNKYgAAAACAjCmJAQAAAAAypiQGAAAAAMiYkhgAAAAAIGMdrT6A2Msvvxxmxo0bVzkfMGBAQ24ZOnRo5bxv377hjt133z3M9OzZM8wMGTIkzEQOPPDAMHP99ddXzj/3uc+FO7q6umq+Cd7N6NGj696xbt26MLNq1aq6nwPQaOeee27lfMcddwx3LFq0KMx0dNT/4/EOO+wQZm655ZYwc/jhh1fOx48fH+546aWXwgw0wxNPPBFm9thjj7qf861vfSvMvPLKK3U/p7vp169fmNl7770r57X82m699dZhZpNNNgkzjbB06dLK+bJly5pyB+2tLMswc//99zfhkjzV8usfZWrZkQtfEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABnraPUBNMbs2bNbfUJD9ezZM8yMGjWqcv6lL30p3LHXXnuFmWOOOaZyPmzYsHDHI488EmYgsueee9a949e//nWY+e53vxtmNtlkk7pv2Zj06tWr1Sf8l+id8+ijjzbpEmiej33sY2Fm4sSJlfNVq1aFO0444YQwM2/evMp53759wx033XRTmIl+jkkppdGjR1fOBwwYEO448cQTw8yLL74YZqBekydPDjMzZ86s+zmnnnpqmOndu3eYWbFiReX8iiuuCHcMHjw4zBx77LFhphE++MEPhpkjjzyyCZc0RldXV5g56aSTKud+jwbd36BBg1p9QlvxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJCxjlYf0O569+5dOV+zZk2TLule1q5dG2amT59eOb/33nvDHV1dXTXf9G6OOOKIMPPII4/U/RxohB122CHMvPnmm024hHfz1ltvtfoEaLpJkyaFmbIsK+ff/va3wx3z5s2r+aZ389prr4WZz372s3U/J6WURo0aVTk/5JBDwh1HH310mPnGN75R60nQFkaPHl33jujPz5RS6tmzZ5jZaqut6r4lRyeccEKY+clPftKES4ANZZtttgkzp556at3Peeyxx+re0S58SQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZ62j1Ad3ZZz7zmTAzfvz4yvmnP/3pcMeLL75Y80380Zo1a1p9AjTUSSedFGZmz55dOd92220bdQ7vwYQJE8LMlVde2YRLoHkGDx4cZg488MAw8+qrr1bOL7300lpP2uDGjh0bZoYPH173c1asWBFmfvWrX9X9HGiE+fPnh5lf/OIXYWavvfZqxDl169evX6tP6LZ+85vfVM6vvfbacMeTTz7ZqHOAjdTkyZPDzMCBA+t+zuWXX173jnbhS2IAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjHa0+oDs79thjw8y6desq58uWLWvUOUCbmzdvXpjZb7/9KucDBgwId4wfP77mm7qDGTNmhJmjjjoqzJx00kmV8+985zvhjquvvjrMrF+/PsxAd7L99tuHmd69e4eZ2267rXK+dOnSmm+qx5FHHhlmLrvssjCzxRZbhJkHH3ywcn722WeHO5555pkwA82wcOHCMDNy5Mgw88ADD1TOhwwZUvNN/HmWLFkSZu68884wE/3MtGjRolpPgg1uzpw5lfPXXnst3HHggQeGmZ/97Gc139QdbLfddmFm0qRJlfPTTjutIbcsXry4cv7UU0815DntwJfEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxjpafUB3dtBBB4WZ3/3ud5XznXfeOdwxf/78mm/KSWdnZ+V82rRp4Y73vS/++yTr16+vnL/++uvhDmiWl156qa55Sikde+yxjTqn26jlfR5ZuXJlmFm3bl3dz4GNTc+ePSvn119/fUOe8+Mf/7hyvvnmm4c7Tj/99DBz9tlnV84HDRoU7qjlZ4PjjjsuzMyePbtyvnbt2nAHdCdLly4NM0ceeWTl/IwzzmjUOZXGjBkTZj7wgQ804ZKUvve974WZBQsW1P2ciy++OMzU8vMQtJOyLMPMeeedF2Z++tOfVs4XLlxY801V9t1338r5sGHDwh2nnHJKmNl6663DTP/+/SvntfzarlmzJsxEv3YrVqwId+TCl8QAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQsaIsy+Y9rCia97AmuPnmm8PMaaedVjl/6aWXwh3nnntumHnxxRcr5z//+c/DHRuTffbZJ8xMnTq1cv6Rj3wk3FEURZh56KGHKucjR44Md6xcuTLMdCdlWca/cBuBdnvnsOFce+21Yebss8+unNfy14Qzzzyz5pv4o+7yzkkpz/fOlltuWTl/4403GvKcZ555pnI+aNCgcMdmm21W9x1Lly4NM4cffniYefbZZ+u+hQ2nu7x3cnznbCz23HPPMLP55ps34ZKUFixYEGaWL1/ehEt4r7xzuq+o80kp7i5SiruJZnZ3kVp6lGbde9FFF4WZyZMnN+GS7uXd3jm+JAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADLW0eoDurPZs2eHmVGjRlXOBwwYEO6YMWNGmHnrrbcq5ytWrAh31OKXv/xl5bwoinDHhz/84TDT2dnZkEzkhRdeCDOnnXZa5XzlypV13wG0VldXV6tPgG4r+uvgV77ylXDHMcccE2b23Xffmm96Nz/84Q/DzLPPPls5nzJlSrhj+fLlNd8EdE9PPvlkq08ANgL33ntvmJkwYUKY2WGHHRpxTlO8/PLLYeaNN94IMw888EDlvJZf24cffjjMUDtfEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABkryrJs3sOKonkP20j079+/cn7WWWeFO77yla+EmWb+caxSFEWYadStb7zxRuV82rRp4Y7rrrsuzDz33HM135SLsizjP9AbgRzfObw3m2yySZh58803K+fHHHNMuOP++++v+Sb+qLu8c1Ly3oF20V3eO9450B68c9rbsGHDwswNN9xQOe/Vq1e4Y5tttgkzc+fOrZwvWrQo3HH77beHmWXLloUZWufd3jm+JAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMFWVZNu9hRdG8h7WRj3/842HmuOOOq5wPHTo03PHaa6+Fmb/5m7+pnBdFEe5YvHhxmPnRj34UZu69997K+YMPPhju4L0pyzL+A70R8M6B9tBd3jkpee9Au+gu7x3vHGgP3jlAM73bO8eXxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaKsiyb97CiaN7DgA2mLMui1TfUwjsH2kN3eeek5L0D7aK7vHe8c6A9eOcAzfRu7xxfEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZKwoy7LVNwAAAAAA0CK+JAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSmIYoiuJDRVE8WBTFb4uiWFwUxadbfRPQvoqiGFwUxfeLoni9KIpXiqK4oSiKjlbfBbSnoii+WBTFvKIo3iqK4vZW3wPkoyiKXYqiWFMUxV2tvgVoX37WISUlMQ3wh2JmTkrpeymlvimlz6eU7iqKYkhLDwPa2ddTSstSStumlPZMKR2cUjqzpRcB7eyllNIlKaVvtfoQIDs3ppQea/URQNvzsw5KYhpit5TSgJTSV8uy/H1Zlg+mlP53SulzrT0LaGM7pJT+qSzLNWVZvpJS+kFK6cMtvgloU2VZzirLcnZK6TetvgXIR1EUx6eU3kgpzW31LUB787MOKSmJaYziXf5/Q5t9CJCNr6WUji+KorMoioEppU+mt4tiAIBuryiKLVNKk1NK57b6FgDyoCSmEZ5Nb//Xvr9cFEXPoiiOSG//V787W3sW0Mb+V3r7y+EVKaUXU0rzUkqzW3oRAEDjXJxS+mZZlr9u9SEA5EFJTN3KslybUhqZUjoypfRKevvvdv9Teru4AWiooijel1L6YUppVkpps5RSv5TS1imly1t5FwBAIxRFsWdK6bCU0ldbfQsA+fBPgqchyrJ8Or399XBKKaWiKH6WUrqjdRcBbaxvSumDKaUbyrJ8K6X0VlEUt6W3/0EL41t6GQBA/Q5JKQ1OKb1QFEVKKW2eUupRFMXuZVnu1cK7AGhjviSmIYqi+EhRFL3/8L8P+j9TStumlG5v8VlAGyrL8tWU0v9JKX2hKIqOoij6pJROSik91drLgHb1h3dN75RSj/R2UdO7KAofWwAbytSU0k4ppT3/8K+bUkr3p5SGt/IooH35WYeUlMQ0zudSSi+nt/+3iT+RUjr8D1/4AWwI/yOlNCKltDyltDiltC6ldE5LLwLa2cSU0psppX9IKY35w/89saUXAW2rLMvVZVm+8v/+lVJalVJaU5bl8lbfBrQtP+uQirIsW30DAAAAAAAt4ktiAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgYx3NfFhRFP4pedAGyrIsWn1DLbxzoD10l3dOSt470C66y3vHOwfag3cO0Ezv9s7xJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMY6Wn0AALSD3r17h5ljjz02zFxwwQWV8x133LHmm6qceeaZlfObb7453FGWZUNuAQAA+HONGDEizMyYMaNyvvPOO4c7Xn311Zpv6s58SQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZK8qybN7DiqJ5DwM2mLIsi1bfUAvvHGpVFPF/pHfaaafK+ezZs8Md2267bZjZdNNNK+e9evUKdzTCkCFDwsxzzz3XhEu6zzsnJe8dajdmzJgwc+edd9b9nC233DLMrFq1qu7ntJvu8t7xzun+Ojs7K+erV69u0iWx6NaUUlqyZEmY+f73v185P/nkk2u8qH1458A7HXbYYWFm5syZYWaLLbaonF9wwQXhjksuuSTMdCfv9s7xJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJCxjlYfQHvZb7/9wswNN9wQZvbdd98wM2nSpMr5xRdfHO4oyzLMAN1b3759w8zxxx8fZq699trK+eOPPx7uOOqoo8LMLrvsUjkfOHBguGPcuHFhZo899qicn3322eGOv//7vw8zkKNhw4aFmalTp4YZP6dA93bwwQeHmWnTplXODzjggHDH888/X/NN9Rg1alSY6devX5g56KCDGnEO0M317t27cn7uueeGO7bYYosws2bNmsr5K6+8Eu7IhS+JAQAAAAAypiQGAAAAAMiYkhgAAAAAIGNKYgAAAACAjCmJAQAAAAAypiQGAAAAAMiYkhgAAAAAIGNKYgAAAACAjHW0+gC6lxEjRlTOL7/88nDH0KFDw8z69evDzKRJkyrnV155ZbjjzTffDDPAO2222WaV8yFDhoQ7Tj311DCzzTbbVM7Hjh0b7vjBD34QZvr37x9mPv3pT1fO77///nBHLZYsWVL3jqVLl4aZ++67r+7nQI4OOuigMDNhwoQw06tXr7pvWbRoUZhZt25d3c8B3qlnz55h5rzzzgszq1evrpwvX7685pvq1adPn8p5Lf9+iqIIM48//njNNwHt6+STT66cDx8+vCHP6erqqpzfeuutDXlOO/AlMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGSso9UH0BxFUYSZb37zm2Fm1KhRlfPNNtus5ps2tJNPPjnMfOMb39jwh0A386EPfSjMTJw4sXJ+/PHHhzteffXVMPP6669Xzu+5555wx+rVq8PMYYcdFmaee+65MLOxePLJJ8PM8uXLm3AJdD/9+vWrnE+dOjXcscsuuzTqnEp33313mFmzZk0TLoH8fPKTnwwzw4cPDzM33HBD5byWn2MaZffdd6+c77bbbuGOVatWhZlrrrmm5puA7mnXXXcNM+PHj6/7OW+++WaY+eIXv1j3c3LhS2IAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjHa0+gMbo3bt35fzmm28Od4wZM6ZR52wUttxyy1afABudzs7OMFPL++KAAw6onH/nO98Jd1x66aVhZtCgQZXzj33sY+GOyZMnh5n169eHme7k1ltvDTMDBw6snP/qV79q1DnQrRxxxBGV81122aVJl6S0du3ayvndd9/dpEuA/98tt9wSZlavXh1mavm5q1mmTJlS947rr78+zPz85z+v+zlA64wYMSLM3HjjjWFm8ODBdd/y2GOPhRk/L9XOl8QAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGOlp9AI1x1FFHVc7HjBnTpEuAjdnAgQPDzAEHHBBmfvGLX1TOzzjjjHBHV1dXmHn22Wcr5z/60Y/CHe3m0EMPDTPDhg0LM3PmzKmcf/3rX6/5Jmgnjz32WOX8xRdfDHdst912DbnlnnvuqZzPnz+/Ic8B3ulrX/ta5bx///7hjjvvvDPMNOvP41GjRoWZgw46qHK+bNmycMcll1xS801A93T88ceHmR122KHu5zz66KNh5vDDD6/7OfyRL4kBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgYx2tPoDYpptuGmbGjx/fhEsA3rZ69erKeVdXV5MuaT+nnXZa5fy6664Ld7zyyith5vzzz6+cr1u3LtwB7WjRokWV89/85jfhju22264ht/zbv/1bQ/YA/91hhx0WZk499dTK+W9/+9twxxVXXFHzTfXo2bNnmLnyyivDTFEUlfN//Md/DHdEPyMCG7dzzjknzIzE4DetAAAOTklEQVQePbohz1q6dGnl/MQTTwx3rF27tiG38DZfEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABnraPUBxMaNGxdm9t577yZcAnR3b7zxRphZvHhxmNl///0r52eddVa448Ybbwwz3UmfPn3CzKxZs8LMRz/60cr5yy+/HO741Kc+FWYWLFgQZoANZ9WqVWHmqaeeasIlkJ8pU6aEmc7Ozsr5l7/85XDH/Pnza76pHieeeGKYGTRoUJhZtmxZ5XzmzJk13wRsnLbddtvKeS2/j+voaEyVOGPGjMr5woULG/IcaudLYgAAAACAjCmJAQAAAAAypiQGAAAAAMiYkhgAAAAAIGNKYgAAAACAjCmJAQAAAAAypiQGAAAAAMiYkhgAAAAAIGMdrT6A9vL888+HmQEDBoSZnj171n3LPffcU/cOaDfLly8PM08++WSY2XnnnSvnX/3qV8MdP/3pT8PM008/HWYaoZZ3zoUXXlg5HzduXLijs7MzzFx11VWV80mTJoU71qxZE2aAP23kyJGV8913370hz/ntb38bZv71X/+1Ic8C/rt99tknzMyfP79yfs011zTqnEq1vHMuu+yyMFMURZiZMGFC5bx///7hjj59+oSZJUuWhBlgw7jkkksq5zvuuGNDnvPQQw+FmYsvvrghz6JxfEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGeto9QHErrvuujCzevXqyvmUKVPCHe97X/z3DG655ZbK+SWXXBLuWLRoUZjp2bNnmJk+fXrl/Lnnngt3AO903HHHhZk5c+ZUzo8++uhwx4wZM8LM5MmTK+ezZs0Kd0yYMCHMHHHEEWFm//33r5x3dXWFO66++uowc95554UZYMPp7OysnNfyM0otevXqFWYGDRpUOX/hhRcacgu0k4MPPjjMlGUZZubPn185/+QnP1nzTVUGDx5cOf/85z8f7ujXr1+YqeXfc/R7vVp2/PM//3OYGTlyZJgB/nwHHnhgmBk1alTlvCiKcMfrr78eZmr5PdiKFSvCDM3lS2IAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjRVmWzXtYUTTvYfw3gwcPDjNbbbVVmHnqqacq55/73OfCHbfffnuYqcXo0aMr53fffXdDnsM7lWVZtPqGWnjnbDgDBgyonH//+98PdwwbNqzuOx5//PEws/fee4eZrq6uMPPtb3+7cn7WWWeFO3hvuss7JyXvnXbw4IMPVs4PPvjghjxn7dq1YeaUU06pnE+fPr0ht/BO3eW9453zTv379w8zjz76aJjZfvvtK+dFEf9HpJbfa0d7mvn79cWLF1fOr7nmmnDHTTfd1KhzsuKdQ2TzzTcPM0uWLAkzffv2rfuWWnqfadOm1f0cNpx3e+f4khgAAAAAIGNKYgAAAACAjCmJAQAAAAAypiQGAAAAAMiYkhgAAAAAIGNKYgAAAACAjCmJAQAAAAAypiQGAAAAAMhYR6sPoDmWLFnS6hMabsGCBa0+AbJVFEXl/Prrrw93TJ06te479t5777p3pJTSxz/+8TDz9NNPN+RZQOuMGDEizOy77751P2ft2rVh5rLLLgsz06dPr/sWyM3y5cvDzPnnnx9mbrnllsr5ZpttFu4oyzLMNGLHM888E2ZGjx4dZhYvXlw5X716dbgDeG969+5dOZ8zZ064o2/fvmEmeqdcc8014Y5p06aFGbonXxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZ62j1AbSX/fbbryF71q5dG2bWrVvXkGdBO+noqH6tDxs2LNwxcuTIMHP66adXzj/wgQ+EOxYsWBBm3nzzzcr50KFDwx3Rr0lKKe28885h5umnnw4zwMZt1113DTOdnZ11P+c//uM/wsxFF11U93OA92bGjBlhZu7cuZXz3XffPdwxYsSIMPMP//APYSby5S9/Ocz4OQY2jEMOOSTMPPTQQ2Em+hnlr//6r2u8qNrll19eOZ8wYUJDnkP35EtiAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyFhHqw+ge9l8880r54ceemhDnvPoo4+GmV/+8pcNeRa0ky9+8YuV86uvvrohz1mxYkXl/I477gh3/N3f/V2YWblyZeV8/Pjx4Y5LLrkkzFx44YVh5pFHHqmcv/TSS+EOoLUa9XMK0P6WL19eOZ8/f36449577w0zZVlWzmv5PU90K7Dh/OxnP2vInlp+X9MIt912W1OeQ/fkS2IAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjHa0+gO7lwx/+cOV8t912a8hz7r777obsgXbSo0ePMHPIIYfU/ZwVK1aEmalTp1bOzzvvvLrvqMUVV1wRZiZOnBhmhg4dGmb+9m//tnJ+/fXXhzuA1ho+fHhTnnPBBRc05TlA6+yxxx5hpk+fPmGmq6urcj527Nhwx7x588IMsGH87ne/CzO77rprmBk5cmTdt6xduzbMrF+/vu7n0L58SQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGSso9UH0L2MHTu27h2vvvpqmLnpppvqfg60m0033TTMHH300XU/Z+LEiWHmxhtvrPs5jbDLLruEmY4Of6mDHHzpS18KM414Hzz88MNhZu7cuXU/B9i4fe1rXwszZVmGmeeff75yvnr16ppvAjZOu+22W5ip5fd6kcsvvzzMPPfcc3U/h/blS2IAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADIWEerD2Djseuuu4aZT3ziE3U/54UXXggzv//97+t+DrSbdevWhZlnnnmmcv6hD30o3PGf//mfNd9Uj5EjR4aZgw8+uHI+evTocEevXr3CzCOPPBJm7rjjjjADbDidnZ2V83POOSfcURRF3XdcddVVYWbNmjV1PwdorehnkN133z3csWrVqjBz7LHHVs7nz58f7gBaZ9iwYWHmrrvuasIlKc2cObMpz6F9+ZIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADIWEerD2DjMXv27DCz3Xbb1f2cSy+9tO4dkKM1a9aEmSuuuKJyftttt4U7pk6dGmauu+66MBN5//vfH2Y6Our/y9S8efPCzIUXXhhmVqxYUfctwHvXo0ePyvmAAQMa8pzoz/VFixY15DnAxm3s2LGV87Iswx0zZ84MM/Pnz6/5JmDjM2jQoDCz2Wab1f2crq6uMLNgwYK6n0PefEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkTEkMAAAAAJAxJTEAAAAAQMaUxAAAAAAAGVMSAwAAAABkrKPVB7Dx6Nu3b907urq6wszcuXPrfg7wp82aNatyPmTIkIY854Mf/GDlfMyYMQ15zk033VQ5f/3118MdU6ZMCTOrV6+u+SagvZ100kmV82effbZJlwAbysCBA8PM0UcfXTlfu3ZtuOOGG26o+Sage1q6dGmYmT9/fpjZaaedKuef+cxnwh1r1qwJM1DFl8QAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQsY5WH0B7uffee8PMypUrm3AJ5GnVqlWV84kTJzbljpNOOqkpzwH4czz88MNhZu7cuU24BGilvffeO8x0dnZWzi+//PJwx7x582q+KSf9+/eve8fy5csbcAnU78knnwwzQ4cObcIlUD9fEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABlTEgMAAAAAZExJDAAAAACQMSUxAAAAAEDGlMQAAAAAABnraPUBbDzGjx8fZr71rW9Vzv/93/+9UecAAPyXlStXVs579OjRpEuA7u6+++4LM94pG87y5ctbfQIAf4IviQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwpiQEAAAAAMqYkBgAAAADImJIYAAAAACBjSmIAAAAAgIwVZVk272FF0byHARtMWZZFq2+ohXcOtIfu8s5JyXsH2kV3ee9450B78M4Bmund3jm+JAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADKmJAYAAAAAyJiSGAAAAAAgY0piAAAAAICMKYkBAAAAADJWlGXZ6hsAAAAAAGgRXxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxpTEAAAAAAAZUxIDAAAAAGRMSQwAAAAAkDElMQAAAABAxv4v5Q3pwkwT4WkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x1440 with 25 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter Pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sparsifier():\n",
    "    def __init__(self, meta):\n",
    "        self.meta = meta\n",
    "    \n",
    "    def _compute_sparsity(self, meta):\n",
    "        return meta['final_sparsity'] + (meta['initial_sparsity'] - meta['final_sparsity'])*(1 - (meta['current_step'] - meta['starting_step'])/((meta['ending_step'] - meta['starting_step'])))**3\n",
    "\n",
    "    def prune(self, model, meta):\n",
    "\n",
    "        binary_masks = {}\n",
    "\n",
    "        sparsity = self._compute_sparsity(meta)\n",
    "    \n",
    "        for k, m in enumerate(model.children()):\n",
    "        \n",
    "            if self.meta['pruning_type'] == \"filters\" and isinstance(m, nn.Conv2d):\n",
    "                \n",
    "                weight = m.weight.data.abs().sum(dim=(1,2,3))\n",
    "                y, i = torch.sort(weight)\n",
    "\n",
    "                spars_index = int(weight.shape[0]*sparsity/100)\n",
    "                threshold = y[spars_index]\n",
    "                mask = weight.gt(threshold).float().cuda()\n",
    "\n",
    "                binary_masks[k] = mask.view(-1, 1, 1, 1)\n",
    "                m.weight.data.mul_(mask.view(-1, 1, 1, 1))\n",
    "                m.bias.data.mul_(mask)\n",
    "                \n",
    "            if self.meta['pruning_type'] == 'weights' and isinstance(m, nn.Conv2d): \n",
    "\n",
    "                weight = m.weight.data.view(-1).clone().abs()\n",
    "                y, i = torch.sort(weight)\n",
    "\n",
    "                spars_index = int(weight.shape[0]*sparsity/100)\n",
    "                threshold = y[spars_index]\n",
    "                mask = weight.gt(threshold).float().cuda()\n",
    "                \n",
    "                binary_masks[k] = mask.view(m.weight.data.shape)\n",
    "                m.weight.data.mul_(mask)\n",
    "                \n",
    "            if isinstance(m, nn.BatchNorm2d):\n",
    "                \n",
    "                m.weight.data.mul_(mask)\n",
    "                m.bias.data.mul_(mask)\n",
    "                m.running_mean.mul_(mask)\n",
    "                m.running_var.mul_(mask)    \n",
    "\n",
    "        return binary_masks    \n",
    "        \n",
    "\n",
    "    def applyBinaryMasks(self, model, masks):\n",
    "\n",
    "        for k, m in enumerate(model.children()):\n",
    "\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                mask = masks[k]\n",
    "                m.weight.data.mul_(mask)   \n",
    "                \n",
    "        return model\n",
    "\n",
    "@dataclass\n",
    "class SparsifyCallback(Callback):\n",
    "    learn:Learner\n",
    "        \n",
    "    def __init__(self, meta):\n",
    "        self.meta = meta\n",
    "        self.sparsifier = Sparsifier(self.meta)\n",
    "        self.binary_masks = None\n",
    "    \n",
    "    def on_train_begin(self, **kwargs):\n",
    "        print(f'Pruning of {self.meta[\"pruning_type\"]} until a sparsity of {self.meta[\"final_sparsity\"]}%')\n",
    "        \n",
    "    def on_epoch_end(self, **kwargs):\n",
    "        print(f'Sparsity: {self.sparsifier._compute_sparsity(self.meta):.2f}%')\n",
    "        \n",
    "    def on_batch_begin(self, **kwargs):\n",
    "\n",
    "        if (self.meta['current_step'] - self.meta['starting_step']) % self.meta['span'] == 0 and self.meta['current_step'] > self.meta['starting_step'] and self.meta['current_step'] < self.meta['ending_step']:\n",
    "            self.binary_masks = self.sparsifier.prune(learn.model, self.meta)\n",
    "            \n",
    "        if self.binary_masks:\n",
    "            learn.model = self.sparsifier.applyBinaryMasks(learn.model, self.binary_masks)   \n",
    "            \n",
    "        if self.meta['current_step'] < self.meta['ending_step']:\n",
    "            self.meta['current_step'] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pruning type supported: \"weights\" and \"filters\".\n",
    "\n",
    "prune_meta = {\n",
    "        \"pruning_type\": \"filters\",\n",
    "        \"starting_step\" : 0,\n",
    "        \"current_step\": 0,\n",
    "        \"ending_step\": epochs * np.ceil(len(data.train_ds)/bs),\n",
    "        \"final_sparsity\": 50,\n",
    "        \"initial_sparsity\": 0,\n",
    "        \"span\": 100\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 20, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(20, 50, 5, 1)\n",
    "        self.pool = nn.AdaptiveAvgPool2d((1))\n",
    "        self.fc1 = nn.Linear(50, 25)\n",
    "        self.fc2 = nn.Linear(25, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 4, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(4, 6, 5, 1)\n",
    "        self.pool = nn.AdaptiveAvgPool2d((1))\n",
    "        self.fc1 = nn.Linear(6, 4)\n",
    "        self.fc2 = nn.Linear(4, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 4, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(4, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=6, out_features=4, bias=True)\n",
       "  (fc2): Linear(in_features=4, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Net()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, Net().cuda(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning of filters until a sparsity of 50%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='3', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/3 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='937', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/queues.py\", line 242, in _feed\n",
      "    send_bytes(obj)\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 200, in send_bytes\n",
      "    self._send_bytes(m[offset:offset + size])\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 404, in _send_bytes\n",
      "    self._send(header + buf)\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 368, in _send\n",
      "    n = write(self._handle, buf)\n",
      "BrokenPipeError: [Errno 32] Broken pipe\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/queues.py\", line 242, in _feed\n",
      "    send_bytes(obj)\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 200, in send_bytes\n",
      "    self._send_bytes(m[offset:offset + size])\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 404, in _send_bytes\n",
      "    self._send(header + buf)\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 368, in _send\n",
      "    n = write(self._handle, buf)\n",
      "BrokenPipeError: [Errno 32] Broken pipe\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/queues.py\", line 242, in _feed\n",
      "    send_bytes(obj)\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 200, in send_bytes\n",
      "    self._send_bytes(m[offset:offset + size])\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 404, in _send_bytes\n",
      "    self._send(header + buf)\n",
      "  File \"/home/nathan/anaconda3/envs/DeepLearning/lib/python3.7/multiprocessing/connection.py\", line 368, in _send\n",
      "    n = write(self._handle, buf)\n",
      "BrokenPipeError: [Errno 32] Broken pipe\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-211-d5014ecf2638>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSparsifyCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprune_meta\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_dl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastprogress/fastprogress.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/basic_data.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m         \u001b[0;34m\"Process and returns items from `DataLoader`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdl\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproc_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/basic_data.py\u001b[0m in \u001b[0;36mproc_batch\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mproc_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;34m\"Process batch `b` of `TensorImage`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m         \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtfms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/torch_core.py\u001b[0m in \u001b[0;36mto_device\u001b[0;34m(b, device)\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;34m\"Recursively put `b` on `device`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mifnone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mto_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mis_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mto_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/torch_core.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;34m\"Recursively put `b` on `device`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mifnone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mto_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mis_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mto_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learn.fit(epochs, 1e-3, callbacks=SparsifyCallback(meta=prune_meta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 661,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Conv2d               [4, 24, 24]          304        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [6, 8, 8]            606        True      \n",
       "______________________________________________________________________\n",
       "AdaptiveAvgPool2d    [6, 1, 1]            0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [4]                  28         True      \n",
       "______________________________________________________________________\n",
       "Linear               [10]                 50         True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 988\n",
       "Total trainable params: 988\n",
       "Total non-trainable params: 0"
      ]
     },
     "execution_count": 661,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 680,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Pruner():\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def filters_removed_ix(self, layer):\n",
    "        filters = layer.weight.data.view(layer.out_channels,-1).clone().abs().sum(dim=1)\n",
    "        indices = np.argwhere(filters == 0)\n",
    "        return indices\n",
    "    \n",
    "    def delete_zero_filters(self, layer):\n",
    "        filters = layer.weight.data\n",
    "        biases = layer.bias.data\n",
    "        indices = self.filters_removed_ix(layer)\n",
    "        final_filters = np.delete(filters, indices, axis=0)\n",
    "        final_biases = np.delete(biases, indices, axis=0)\n",
    "        return final_filters, final_biases\n",
    "    \n",
    "    def delete_fc_weights(self, layer, last_conv):\n",
    "        \n",
    "        #input_size = math.sqrt(model.fc1.in_features / (model.conv2.out_channels)) # Gives the size of the feature maps\n",
    "        #new_input_size = input_size**2 * pruner.filters_removed_ix(learn.model.conv2).shape[1] # Give the new size of input of fc layer\n",
    "        last_conv_filters_ix = self.filters_removed_ix(last_conv)\n",
    "    \n",
    "        weights = layer.weight.data\n",
    "        #biases = layer.bias.data\n",
    "    \n",
    "        final_weights = np.delete(weights, last_conv_filters_ix, axis=1)\n",
    "        #final_biases = np.delete(biases, last_conv_filters_ix, axis=0)\n",
    "    \n",
    "        return final_weights        \n",
    "        \n",
    "    def prune_layer(self, model, layer):\n",
    "        # Replace the convolution with a new convolution and new weights\n",
    "        final_filters,_ = self.delete_zero_filters(layer)\n",
    "        layer = nn.Conv2d(layer.in_channels, \n",
    "                          len(final_filters),\n",
    "                          layer.kernel_size,\n",
    "                          layer.stride,\n",
    "                          layer.padding,\n",
    "                          bias=True)\n",
    "        return model\n",
    "    \n",
    "    def _get_last_conv_ix(self, model):\n",
    "        layer_names = list(dict(model.named_children()).keys())\n",
    "        last_conv_ix = 0\n",
    "        for i in range(len(layer_names)):\n",
    "            if getattr(model, layer_names[i]).__class__.__name__ == 'Conv2d':\n",
    "                last_conv_ix = i\n",
    "                \n",
    "        return last_conv_ix\n",
    "    \n",
    "    def _get_first_fc_ix(self, model):\n",
    "        layer_names = list(dict(model.named_children()).keys())\n",
    "        first_fc_ix = 0\n",
    "        for i in range(len(layer_names)):\n",
    "            if getattr(model, layer_names[i]).__class__.__name__ == 'Linear':\n",
    "                first_fc_ix = i\n",
    "                break\n",
    "                \n",
    "        return first_fc_ix        \n",
    "        \n",
    "    \n",
    "    def prune_model(self, model):\n",
    "\n",
    "        pruned_model = copy.deepcopy(model)\n",
    "        layer_names = list(dict(model.named_children()).keys()) # get the names of the layers\n",
    "            \n",
    "        for k, m in enumerate(pruned_model.children()):\n",
    "            \n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                \n",
    "                print(m)\n",
    "                print(getattr(pruned_model, layer_names[k]))\n",
    "                final_filters, final_biases = self.delete_zero_filters(m)\n",
    "                print(final_filters.shape)\n",
    "                    \n",
    "                in_channels = m.in_channels if k==0 else getattr(pruned_model, layer_names[k-1]).out_channels\n",
    "                \n",
    "                new_m = nn.Conv2d(in_channels,\n",
    "                                    len(final_filters),\n",
    "                                    m.kernel_size,\n",
    "                                    m.stride,\n",
    "                                    m.padding,\n",
    "                                    bias=True)\n",
    "                \n",
    "                setattr(new_m, 'weight', nn.Parameter(final_filters))\n",
    "                setattr(new_m, 'bias', nn.Parameter(final_biases))\n",
    "                setattr(pruned_model, layer_names[k], new_m)               \n",
    "                \n",
    "            if isinstance(m, nn.Linear):\n",
    "\n",
    "                last_conv_ix = self._get_last_conv_ix(model)\n",
    "                first_fc_ix = self._get_first_fc_ix(model)\n",
    "                \n",
    "                if k == first_fc_ix:\n",
    "                    final_weights = self.delete_fc_weights(m, getattr(model, layer_names[last_conv_ix]))\n",
    "                    in_channels = final_weights.shape[1]\n",
    "                    new_m = nn.Linear(in_features=in_channels,\n",
    "                                      out_features=m.out_features,\n",
    "                                      bias=True)\n",
    "                \n",
    "                    setattr(new_m, 'weight', nn.Parameter(final_weights))\n",
    "                    #setattr(new_m, 'bias', nn.Parameter(final_biases))                     \n",
    "                    setattr(pruned_model, layer_names[k], new_m)        \n",
    "                else: \n",
    "                    pass\n",
    "                \n",
    "        return pruned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 681,
   "metadata": {},
   "outputs": [],
   "source": [
    "pruner = Pruner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 682,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv2d(3, 4, kernel_size=(5, 5), stride=(1, 1))\n",
      "Conv2d(3, 4, kernel_size=(5, 5), stride=(1, 1))\n",
      "torch.Size([2, 3, 5, 5])\n",
      "Conv2d(4, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "Conv2d(4, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "torch.Size([3, 4, 5, 5])\n"
     ]
    }
   ],
   "source": [
    "pruned_model = pruner.prune_model(learn.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 646,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 2, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(2, 3, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=3, out_features=4, bias=True)\n",
       "  (fc2): Linear(in_features=4, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 646,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pruned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 647,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, pruned_model.cuda(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 648,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 4, 5, 5])"
      ]
     },
     "execution_count": 648,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model.conv2.weight.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 630,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Given groups=1, weight of size 3 4 5 5, expected input[1, 2, 12, 12] to have 4 channels, but got 2 channels instead",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-630-bc39e9e85f86>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/callbacks/hooks.py\u001b[0m in \u001b[0;36mmodel_summary\u001b[0;34m(m, n)\u001b[0m\n\u001b[1;32m    165\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmodel_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m70\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;34m\"Print a summary of `m` using a output text width of `n` chars\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m     \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayers_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m     \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"Layer (type)\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Output Shape\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Param #\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Trainable\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"=\"\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mn\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/callbacks/hooks.py\u001b[0m in \u001b[0;36mlayers_info\u001b[0;34m(m)\u001b[0m\n\u001b[1;32m    159\u001b[0m     \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_layer_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflatten_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m     \u001b[0mlayers_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLearner\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m     \u001b[0mlayers_sizes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers_trainable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparams_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    162\u001b[0m     \u001b[0mlayer_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnamedtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Layer_Information'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'Layer'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'OutputSize'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Params'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Trainable'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer_info\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers_sizes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers_trainable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/fastai/callbacks/hooks.py\u001b[0m in \u001b[0;36mparams_size\u001b[0;34m(m, size)\u001b[0m\n\u001b[1;32m    147\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mhook_outputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflatten_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mhook_o\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mhook_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflatten_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;32mas\u001b[0m \u001b[0mhook_p\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    150\u001b[0m             \u001b[0moutput_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstored\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstored\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhook_o\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m             \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstored\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstored\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhook_p\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-552-aa4736ed9cc0>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    336\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[1;32m    337\u001b[0m         return F.conv2d(input, self.weight, self.bias, self.stride,\n\u001b[0;32m--> 338\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    339\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Given groups=1, weight of size 3 4 5 5, expected input[1, 2, 12, 12] to have 4 channels, but got 2 channels instead"
     ]
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2.3052714, tensor(0.1413)]"
      ]
     },
     "execution_count": 452,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [],
   "source": [
    "                    new_m = nn.Linear(in_features=16,\n",
    "                                      out_features=10,\n",
    "                                      bias=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10])"
      ]
     },
     "execution_count": 544,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_m.bias.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 20, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(20, 50, 5, 1)\n",
    "        self.pool = nn.AdaptiveAvgPool2d((4))\n",
    "        self.fc1 = nn.Linear(800, 100)\n",
    "        self.fc2 = nn.Linear(100, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test when the output of global average pooling > 1x1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 20, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(20, 4, 5, 1)\n",
    "        self.pool = nn.AdaptiveAvgPool2d((2))\n",
    "        self.fc1 = nn.Linear(16, 4)\n",
    "        self.fc2 = nn.Linear(4, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pruning type supported: \"weights\" and \"filters\".\n",
    "\n",
    "prune_meta = {\n",
    "        \"pruning_type\": \"filters\",\n",
    "        \"starting_step\" : 0,\n",
    "        \"current_step\": 0,\n",
    "        \"ending_step\": epochs * np.ceil(len(data.train_ds)/bs),\n",
    "        \"final_sparsity\": 50,\n",
    "        \"initial_sparsity\": 0,\n",
    "        \"span\": 100\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, Net().cuda(), metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning of filters until a sparsity of 50%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.319383</td>\n",
       "      <td>1.291876</td>\n",
       "      <td>0.557400</td>\n",
       "      <td>00:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.841015</td>\n",
       "      <td>0.773434</td>\n",
       "      <td>0.753400</td>\n",
       "      <td>00:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.599839</td>\n",
       "      <td>0.542658</td>\n",
       "      <td>0.840900</td>\n",
       "      <td>00:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparsity: 38.58%\n",
      "Sparsity: 49.45%\n",
      "Sparsity: 50.00%\n"
     ]
    }
   ],
   "source": [
    "learn.fit(epochs, 1e-3, callbacks=SparsifyCallback(meta = prune_meta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 20, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(20, 4, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=2)\n",
       "  (fc1): Linear(in_features=16, out_features=4, bias=True)\n",
       "  (fc2): Linear(in_features=4, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 392,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Conv2d               [20, 24, 24]         1,520      True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [4, 8, 8]            2,004      True      \n",
       "______________________________________________________________________\n",
       "AdaptiveAvgPool2d    [4, 2, 2]            0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [4]                  68         True      \n",
       "______________________________________________________________________\n",
       "Linear               [10]                 50         True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 3,642\n",
       "Total trainable params: 3,642\n",
       "Total non-trainable params: 0"
      ]
     },
     "execution_count": 418,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def delete_fc_weights(layer, last_conv):\n",
    "        \n",
    "        input_size = math.sqrt(layer.in_features / (last_conv.out_channels)) # Gives the size of the feature maps\n",
    "        #print(input_size)\n",
    "        #new_input_size = input_size**2 * pruner.filters_removed_ix(last_conv).shape[1] # Give the new size after pooling\n",
    "        #print(new_input_size)\n",
    "        last_conv_filters_ix = pruner.filters_removed_ix(last_conv) # Get the index of the filters removed\n",
    "    \n",
    "        weights = layer.weight.data\n",
    "        biases = layer.bias.data\n",
    "        \n",
    "        start_weights_ix = last_conv_filters_ix*input_size\n",
    "        end_weights_ix = last_conv_filters_ix*input_size + input_size\n",
    "        print(start_weights_ix)\n",
    "        print(end_weights_ix)\n",
    "        \n",
    "        final_weights = np.delete(weights, last_conv_filters_ix, axis=-1)\n",
    "        final_biases = np.delete(biases, last_conv_filters_ix, axis=0)\n",
    "    \n",
    "        return final_weights, final_biases "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = learn.model.fc1.weight.data\n",
    "biases = learn.model.fc1.bias.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 2]])"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_conv_filters_ix = pruner.filters_removed_ix(learn.model.conv2); last_conv_filters_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 16])"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0710,  0.2370,  0.0483,  0.0808], device='cuda:0')"
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = learn.model.fc1\n",
    "last_conv = learn.model.conv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0, 4]])\n",
      "tensor([[2, 6]])\n"
     ]
    }
   ],
   "source": [
    "final_w, final_b = delete_fc_weights(learn.model.fc1, learn.model.conv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0, 4]])\n",
      "tensor([[2, 6]])\n"
     ]
    }
   ],
   "source": [
    "        input_size = math.sqrt(layer.in_features / (last_conv.out_channels)) # Gives the size of the feature maps\n",
    "        #print(input_size)\n",
    "        #new_input_size = input_size**2 * pruner.filters_removed_ix(last_conv).shape[1] # Give the new size after pooling\n",
    "        #print(new_input_size)\n",
    "        last_conv_filters_ix = pruner.filters_removed_ix(last_conv) # Get the index of the filters removed\n",
    "    \n",
    "        weights = layer.weight.data\n",
    "        biases = layer.bias.data\n",
    "        \n",
    "        start_weights_ix = last_conv_filters_ix*input_size\n",
    "        end_weights_ix = last_conv_filters_ix*input_size + input_size\n",
    "        print(start_weights_ix)\n",
    "        print(end_weights_ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "only one element tensors can be converted to Python scalars",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-425-b453d386abae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfinal_weights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstart_weights_ix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_weights_ix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36mdelete\u001b[0;34m(arr, obj, axis)\u001b[0m\n\u001b[1;32m   4360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4361\u001b[0m     \u001b[0m_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4362\u001b[0;31m     \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4363\u001b[0m     \u001b[0;31m# After removing the special handling of booleans and out of\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4364\u001b[0m     \u001b[0;31m# bounds values, the conversion to the array can be removed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/DeepLearning/lib/python3.7/site-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m    536\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m     \"\"\"\n\u001b[0;32m--> 538\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: only one element tensors can be converted to Python scalars"
     ]
    }
   ],
   "source": [
    "final_weights = np.delete(weights, [start_weights_ix, end_weights_ix], axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pruner 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 4, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(4, 6, 5, 1)\n",
    "        self.pool = nn.AdaptiveAvgPool2d((1))\n",
    "        self.fc1 = nn.Linear(6, 4)\n",
    "        self.fc2 = nn.Linear(4, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, mnist=True):\n",
    "        super().__init__()\n",
    "          \n",
    "        self.conv1 = nn.Conv2d(3, 4, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(4, 6, 5, 1)\n",
    "        self.conv3 = nn.Conv2d(6,4, 5, 1)\n",
    "        self.pool = nn.AdaptiveAvgPool2d((1))\n",
    "        self.fc1 = nn.Linear(4, 6)\n",
    "        self.fc2 = nn.Linear(6, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 4, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(4, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv3): Conv2d(6, 4, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=4, out_features=6, bias=True)\n",
       "  (fc2): Linear(in_features=6, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Net()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The final net should have:\n",
    "- conv1: Conv2d(3,2)\n",
    "- conv2: Conv2d(2,3)\n",
    "- fc1: Linear(3, 4)\n",
    "- fc2: Linear(4,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pruning type supported: \"weights\" and \"filters\".\n",
    "prune_meta = {\n",
    "        \"pruning_type\": \"filters\",\n",
    "        \"starting_step\" : 0,\n",
    "        \"current_step\": 0,\n",
    "        \"ending_step\": epochs * np.ceil(len(data.train_ds)/bs),\n",
    "        \"final_sparsity\": 50,\n",
    "        \"initial_sparsity\": 0,\n",
    "        \"span\": 100\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning of filters until a sparsity of 50%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.242686</td>\n",
       "      <td>1.200006</td>\n",
       "      <td>0.577900</td>\n",
       "      <td>00:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.923993</td>\n",
       "      <td>0.934824</td>\n",
       "      <td>0.678200</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.852159</td>\n",
       "      <td>0.823015</td>\n",
       "      <td>0.715900</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparsity: 38.58%\n",
      "Sparsity: 49.45%\n",
      "Sparsity: 50.00%\n"
     ]
    }
   ],
   "source": [
    "learn = Learner(data, Net().cuda(), metrics=accuracy)\n",
    "\n",
    "learn.fit(epochs, 1e-3, callbacks=SparsifyCallback(meta=prune_meta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_names = list(dict(learn.model.named_children()).keys()) # get the names of the layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def filters_to_keep(layer, nxt_layer):\n",
    "    \n",
    "        is_cuda = layer.weight.is_cuda\n",
    "    \n",
    "        filters = layer.weight\n",
    "        biases = layer.bias\n",
    "        nz_filters = filters.data.view(layer.out_channels, -1).sum(dim=1) # Flatten the filters to compare them\n",
    "        ixs = torch.LongTensor(np.argwhere(nz_filters!=0)) # Get which filters are not equal to zero\n",
    "\n",
    "        ixs = ixs.cuda() if is_cuda else ixs\n",
    "    \n",
    "        filters_keep = filters.index_select(0, ixs[0]).data # keep only the non_zero filters\n",
    "        biases_keep = biases.index_select(0, ixs[0]).data\n",
    "        \n",
    "        #print(filters_keep.shape)\n",
    "        return filters_keep, biases_keep, nxt_filters_keep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_conv(layer):\n",
    "    assert layer.__class__.__name__ == 'Conv2d'\n",
    "    \n",
    "    new_weights, new_biases = filters_to_keep(layer)\n",
    "\n",
    "    new_out_channels = new_weights.shape[0]\n",
    "    new_in_channels = new_weights.shape[1]\n",
    "    \n",
    "    layer.weight = nn.Parameter(new_weights)\n",
    "    layer.biases = nn.Parameter(new_biases)\n",
    "    \n",
    "    layer.out_channels = new_out_channels\n",
    "    layer.in_channels = new_in_channels\n",
    "    \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_next_conv(model, conv_ix):\n",
    "    for k,m in enumerate(model.children()):\n",
    "        if k > conv_ix and m.__class__.__name__ == 'Conv2d':\n",
    "            next_conv_ix = k\n",
    "            break\n",
    "        else:\n",
    "            next_conv_ix = None\n",
    "            \n",
    "    return next_conv_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_next_conv(learn.model, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_input(layer, next_layer):\n",
    "    \n",
    "    next_layer.in_channels = layer.out_channels\n",
    "    \n",
    "    return next_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def prune_model(model):\n",
    "        pruned_model = copy.deepcopy(model)\n",
    "        \n",
    "        layer_names = list(dict(pruned_model.named_children()).keys())\n",
    "        #print(layer_names)\n",
    "        \n",
    "        for k,m in enumerate(list(pruned_model.children())):\n",
    "            \n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                next_conv_ix = find_next_conv(model, k)\n",
    "                if next_conv_ix is not None: # The conv layer is not the last one\n",
    "\n",
    "                    new_m = prune_conv(m) # Prune the current conv layer\n",
    "                    setattr(pruned_model, layer_names[k], new_m) # Apply the changes to the model\n",
    "\n",
    "                    next_conv = getattr(pruned_model, layer_names[next_conv_ix]) # Get the next_conv_layer\n",
    "                    new_next_conv = change_input(m, next_conv) # Change its number of in_channels\n",
    "                    \n",
    "                    print(next_conv.in_channels)\n",
    "                    print(pruned_model)\n",
    "                    setattr(pruned_model, layer_names[next_conv_ix], new_next_conv)\n",
    "                else:\n",
    "                    print('last conv')\n",
    "                    #new_m = prune_conv(m) # Prune the current conv layer\n",
    "                    #setattr(pruned_model, layer_names[k], new_m) # Apply the changes to the model\n",
    "    \n",
    "            if isinstance(m, nn.Linear):\n",
    "\n",
    "                last_conv_ix = self._get_last_conv_ix(model)\n",
    "                first_fc_ix = self._get_first_fc_ix(model)\n",
    "                \n",
    "                if k == first_fc_ix:\n",
    "                    final_weights = self.delete_fc_weights(m, getattr(model, layer_names[last_conv_ix]))\n",
    "                    in_channels = final_weights.shape[1]\n",
    "                    new_m = nn.Linear(in_features=in_channels,\n",
    "                                      out_features=m.out_features,\n",
    "                                      bias=True)\n",
    "                \n",
    "                    setattr(new_m, 'weight', nn.Parameter(final_weights))\n",
    "                    #setattr(new_m, 'bias', nn.Parameter(final_biases))                     \n",
    "                    setattr(pruned_model, layer_names[k], new_m)        \n",
    "                else: \n",
    "                    pass                  \n",
    "        return pruned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 4, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(4, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv3): Conv2d(6, 4, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=4, out_features=6, bias=True)\n",
       "  (fc2): Linear(in_features=6, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 6, 5, 5])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model.conv3.weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.3867,  0.5445,  0.2913,  0.3530],\n",
       "        [-0.0923,  0.3431, -0.4521,  0.5386],\n",
       "        [-0.1764, -0.1153,  0.1784,  0.3180],\n",
       "        [-0.0106, -0.2108,  0.0665, -0.1509],\n",
       "        [-0.0974, -0.0839,  0.2173,  0.5620],\n",
       "        [ 0.7335,  0.3001, -0.1628, -0.0173]], device='cuda:0',\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model.fc1.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "prune_conv() missing 1 required positional argument: 'nxt_layer'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-496-9a6064d1c632>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprune_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-495-59d95c86f8df>\u001b[0m in \u001b[0;36mprune_model\u001b[0;34m(model)\u001b[0m\n\u001b[1;32m     11\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mnext_conv_ix\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# The conv layer is not the last one\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m                 \u001b[0mnew_m\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprune_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Prune the current conv layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m                 \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpruned_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_names\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_m\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Apply the changes to the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: prune_conv() missing 1 required positional argument: 'nxt_layer'"
     ]
    }
   ],
   "source": [
    "m = prune_model(learn.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 2, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(4, 3, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv3): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=6, out_features=4, bias=True)\n",
       "  (fc2): Linear(in_features=4, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def filters_to_keep(layer, nxt_layer):\n",
    "    \n",
    "        is_cuda = layer.weight.is_cuda\n",
    "    \n",
    "        filters = layer.weight\n",
    "        biases = layer.bias\n",
    "        nz_filters = filters.data.view(layer.out_channels, -1).sum(dim=1) # Flatten the filters to compare them\n",
    "        ixs = torch.LongTensor(np.argwhere(nz_filters!=0)) # Get which filters are not equal to zero\n",
    "\n",
    "        ixs = ixs.cuda() if is_cuda else ixs\n",
    "    \n",
    "        filters_keep = filters.index_select(0, ixs[0]).data # keep only the non_zero filters\n",
    "        biases_keep = biases.index_select(0, ixs[0]).data\n",
    "        \n",
    "        if nxt_layer is not None:\n",
    "            nxt_filters = nxt_layer.weight\n",
    "            nxt_filters_keep = nxt_filters.index_select(1, ixs[0]).data\n",
    "        else:\n",
    "            nxt_filters_keep = None\n",
    "            \n",
    "        return filters_keep, biases_keep, nxt_filters_keep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_conv(layer, nxt_layer):\n",
    "    assert layer.__class__.__name__ == 'Conv2d'\n",
    "    \n",
    "    new_weights, new_biases, new_next_weights = filters_to_keep(layer, nxt_layer)\n",
    "\n",
    "    new_out_channels = new_weights.shape[0]\n",
    "    new_in_channels = new_weights.shape[1]\n",
    "\n",
    "    \n",
    "    layer.out_channels = new_out_channels\n",
    "    layer.in_channels = new_in_channels\n",
    "    \n",
    "    layer.weight = nn.Parameter(new_weights)\n",
    "    layer.bias = nn.Parameter(new_biases)\n",
    "    \n",
    "    \n",
    "    \n",
    "    #print(layer.biases.shape)    \n",
    "    \n",
    "    if new_next_weights is not None:\n",
    "        new_next_in_channels = new_next_weights.shape[1]\n",
    "        nxt_layer.weight = nn.Parameter(new_next_weights)\n",
    "        nxt_layer.in_channels = new_next_in_channels\n",
    "    \n",
    "    return layer, nxt_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def delete_fc_weights(layer, last_conv):\n",
    "        \n",
    "        is_cuda = last_conv.weight.is_cuda\n",
    "\n",
    "        \n",
    "        filters = last_conv.weight\n",
    "        nz_filters = filters.data.view(last_conv.out_channels, -1).sum(dim=1) # Flatten the filters to compare them\n",
    "        ixs = torch.LongTensor(np.argwhere(nz_filters!=0))\n",
    "        \n",
    "        ixs = ixs.cuda() if is_cuda else ixs\n",
    "        \n",
    "        weights = layer.weight.data\n",
    "        \n",
    "        #biases = layer.bias.data\n",
    "        weights_keep = weights.index_select(1, ixs[0]).data\n",
    "        \n",
    "        \n",
    "        layer.in_features = weights_keep.shape[1]\n",
    "        layer.weight = nn.Parameter(weights_keep)\n",
    "    \n",
    "\n",
    "        return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_next_conv(model, conv_ix):\n",
    "    for k,m in enumerate(model.children()):\n",
    "        if k > conv_ix and m.__class__.__name__ == 'Conv2d':\n",
    "            next_conv_ix = k\n",
    "            break\n",
    "        else:\n",
    "            next_conv_ix = None\n",
    "            \n",
    "    return next_conv_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def _get_last_conv_ix(model):\n",
    "        layer_names = list(dict(model.named_children()).keys())\n",
    "        last_conv_ix = 0\n",
    "        for i in range(len(layer_names)):\n",
    "            if getattr(model, layer_names[i]).__class__.__name__ == 'Conv2d':\n",
    "                last_conv_ix = i\n",
    "                \n",
    "        return last_conv_ix\n",
    "    \n",
    "    def _get_first_fc_ix(model):\n",
    "        layer_names = list(dict(model.named_children()).keys())\n",
    "        first_fc_ix = 0\n",
    "        for i in range(len(layer_names)):\n",
    "            if getattr(model, layer_names[i]).__class__.__name__ == 'Linear':\n",
    "                first_fc_ix = i\n",
    "                break\n",
    "                \n",
    "        return first_fc_ix    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_next_conv(learn.model, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_input(layer, next_layer):\n",
    "    \n",
    "    next_layer.in_channels = layer.out_channels\n",
    "    \n",
    "    return next_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def prune_model(model):\n",
    "        pruned_model = copy.deepcopy(model)\n",
    "        \n",
    "        layer_names = list(dict(pruned_model.named_children()).keys())\n",
    "        #print(layer_names)\n",
    "        \n",
    "        for k,m in enumerate(list(pruned_model.children())):\n",
    "            last_conv_ix = _get_last_conv_ix(pruned_model)\n",
    "            first_fc_ix = _get_first_fc_ix(pruned_model)\n",
    "            \n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                next_conv_ix = find_next_conv(model, k)\n",
    "                if next_conv_ix is not None: # The conv layer is not the last one\n",
    "                    next_conv = getattr(pruned_model, layer_names[next_conv_ix]) # Get the next_conv_layer\n",
    "                    new_m, new_next_m = prune_conv(m, next_conv) # Prune the current conv layer\n",
    "                    \n",
    "                    print(new_m.bias.shape)\n",
    "                    setattr(pruned_model, layer_names[k], new_m) # Apply the changes to the model\n",
    "                    #print(pruned_model.conv1.weight.shape)\n",
    "                    setattr(pruned_model, layer_names[next_conv_ix], new_next_m)\n",
    "\n",
    "            #    else:\n",
    "            #        new_m, _ = prune_conv(m, None) # Prune the current conv layer without changing the next one\n",
    "            #        setattr(pruned_model, layer_names[k], new_m) # Apply the changes to the model\n",
    "                    \n",
    "            #if isinstance(m, nn.Linear) and k==first_fc_ix:\n",
    "            #    new_m = delete_fc_weights(m, getattr(pruned_model, layer_names[last_conv_ix]))\n",
    "            \n",
    "            else:\n",
    "                pass\n",
    "        return pruned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5])\n",
      "torch.Size([8])\n"
     ]
    }
   ],
   "source": [
    "m = prune_model(learn.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 10, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(10, 16, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv3): Conv2d(16, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=6, out_features=4, bias=True)\n",
       "  (fc2): Linear(in_features=4, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 5, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv2): Conv2d(5, 8, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (conv3): Conv2d(8, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (fc1): Linear(in_features=6, out_features=4, bias=True)\n",
       "  (fc2): Linear(in_features=4, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_learn = Learner(data, m, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.40331146, tensor(0.8740)]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Pruner():\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def filters_to_keep(self, layer, nxt_layer):\n",
    "    \n",
    "        is_cuda = layer.weight.is_cuda\n",
    "    \n",
    "        filters = layer.weight\n",
    "        biases = layer.bias\n",
    "        nz_filters = filters.data.view(layer.out_channels, -1).sum(dim=1) # Flatten the filters to compare them\n",
    "        ixs = torch.LongTensor(np.argwhere(nz_filters!=0)) # Get which filters are not equal to zero\n",
    "\n",
    "        ixs = ixs.cuda() if is_cuda else ixs\n",
    "    \n",
    "        filters_keep = filters.index_select(0, ixs[0]).data # keep only the non_zero filters\n",
    "        biases_keep = biases.index_select(0, ixs[0]).data\n",
    "        \n",
    "        if nxt_layer is not None:\n",
    "            nxt_filters = nxt_layer.weight\n",
    "            nxt_filters_keep = nxt_filters.index_select(1, ixs[0]).data\n",
    "        else:\n",
    "            nxt_filters_keep = None\n",
    "            \n",
    "        return filters_keep, biases_keep, nxt_filters_keep\n",
    "    \n",
    "    \n",
    "    def prune_conv(self, layer, nxt_layer):\n",
    "        assert layer.__class__.__name__ == 'Conv2d'\n",
    "    \n",
    "        new_weights, new_biases, new_next_weights = self.filters_to_keep(layer, nxt_layer)\n",
    "\n",
    "        new_out_channels = new_weights.shape[0]\n",
    "        new_in_channels = new_weights.shape[1]\n",
    "\n",
    "    \n",
    "        layer.out_channels = new_out_channels\n",
    "        layer.in_channels = new_in_channels\n",
    "    \n",
    "        layer.weight = nn.Parameter(new_weights)\n",
    "        layer.bias = nn.Parameter(new_biases)\n",
    "    \n",
    "\n",
    "        if new_next_weights is not None:\n",
    "            new_next_in_channels = new_next_weights.shape[1]\n",
    "            nxt_layer.weight = nn.Parameter(new_next_weights)\n",
    "            nxt_layer.in_channels = new_next_in_channels\n",
    "    \n",
    "        return layer, nxt_layer\n",
    "\n",
    "    def delete_fc_weights(self, layer, last_conv):\n",
    "        \n",
    "        is_cuda = last_conv.weight.is_cuda\n",
    "\n",
    "        \n",
    "        filters = last_conv.weight\n",
    "        nz_filters = filters.data.view(last_conv.out_channels, -1).sum(dim=1) # Flatten the filters to compare them\n",
    "        ixs = torch.LongTensor(np.argwhere(nz_filters!=0))\n",
    "        \n",
    "        ixs = ixs.cuda() if is_cuda else ixs\n",
    "        \n",
    "        weights = layer.weight.data\n",
    "        \n",
    "        #biases = layer.bias.data\n",
    "        weights_keep = weights.index_select(1, ixs[0]).data\n",
    "        \n",
    "        \n",
    "        layer.in_features = weights_keep.shape[1]\n",
    "        layer.weight = nn.Parameter(weights_keep)\n",
    "    \n",
    "\n",
    "        return layer\n",
    "    \n",
    "    def _find_next_conv(self, model, conv_ix):\n",
    "        for k,m in enumerate(model.children()):\n",
    "            if k > conv_ix and m.__class__.__name__ == 'Conv2d':\n",
    "                next_conv_ix = k\n",
    "                break\n",
    "            else:\n",
    "                next_conv_ix = None\n",
    "            \n",
    "        return next_conv_ix\n",
    "    \n",
    "    def _get_last_conv_ix(self, model):\n",
    "        layer_names = list(dict(model.named_children()).keys())\n",
    "        last_conv_ix = 0\n",
    "        for i in range(len(layer_names)):\n",
    "            if getattr(model, layer_names[i]).__class__.__name__ == 'Conv2d':\n",
    "                last_conv_ix = i\n",
    "                \n",
    "        return last_conv_ix\n",
    "    \n",
    "    def _get_first_fc_ix(self, model):\n",
    "        layer_names = list(dict(model.named_children()).keys())\n",
    "        first_fc_ix = 0\n",
    "        for i in range(len(layer_names)):\n",
    "            if getattr(model, layer_names[i]).__class__.__name__ == 'Linear':\n",
    "                first_fc_ix = i\n",
    "                break\n",
    "                \n",
    "        return first_fc_ix\n",
    "    \n",
    "    def prune_model(self, model):\n",
    "        pruned_model = copy.deepcopy(model)\n",
    "        \n",
    "        layer_names = list(dict(pruned_model.named_children()).keys())\n",
    "        #print(layer_names)\n",
    "        \n",
    "        for k,m in enumerate(list(pruned_model.children())):\n",
    "            last_conv_ix = self._get_last_conv_ix(pruned_model)\n",
    "            first_fc_ix = self._get_first_fc_ix(pruned_model)\n",
    "            \n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                next_conv_ix = self._find_next_conv(model, k)\n",
    "                if next_conv_ix is not None: # The conv layer is not the last one\n",
    "                    next_conv = getattr(pruned_model, layer_names[next_conv_ix]) # Get the next_conv_layer\n",
    "                    new_m, new_next_m = self.prune_conv(m, next_conv) # Prune the current conv layer\n",
    "                    \n",
    "                    print(new_m.bias.shape)\n",
    "                    setattr(pruned_model, layer_names[k], new_m) # Apply the changes to the model\n",
    "                    #print(pruned_model.conv1.weight.shape)\n",
    "                    setattr(pruned_model, layer_names[next_conv_ix], new_next_m)\n",
    "\n",
    "                else:\n",
    "                    new_m, _ = self.prune_conv(m, None) # Prune the current conv layer without changing the next one\n",
    "                    setattr(pruned_model, layer_names[k], new_m) # Apply the changes to the model\n",
    "                    \n",
    "            if isinstance(m, nn.Linear) and k==first_fc_ix:\n",
    "                new_m = self.delete_fc_weights(m, getattr(model, layer_names[last_conv_ix]))\n",
    "            \n",
    "            else:\n",
    "                pass\n",
    "        return pruned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "pruner = Pruner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2])\n",
      "torch.Size([3])\n"
     ]
    }
   ],
   "source": [
    "new_model = pruner.prune_model(learn.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_learn  = Learner(data, new_model, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8230152, tensor(0.7159)]"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8230152, tensor(0.7159)]"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = learn.model.conv3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "        filters = layer.weight\n",
    "        biases = layer.bias\n",
    "        nz_filters = filters.data.view(layer.out_channels, -1).sum(dim=1) # Flatten the filters to compare them\n",
    "        ixs = torch.LongTensor(np.argwhere(nz_filters!=0)) # Get which filters are not equal to zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[[[ 4.4311e-02,  9.8770e-02,  8.1760e-02,  1.1353e-01, -4.7932e-02],\n",
       "          [ 1.8700e-02,  8.0219e-02, -4.0261e-02,  1.6043e-02,  1.1944e-02],\n",
       "          [-7.9537e-02, -2.3543e-02,  5.3704e-02,  4.7743e-02, -7.4830e-02],\n",
       "          [-6.9992e-02, -5.0208e-02,  5.4564e-03, -1.9215e-02,  3.6341e-04],\n",
       "          [-2.3438e-02, -2.0973e-02,  1.3777e-02, -1.1221e-02, -2.2521e-02]],\n",
       "\n",
       "         [[-1.0758e-01, -9.6025e-02, -2.0650e-01, -5.0855e-02,  1.9266e-01],\n",
       "          [-1.6935e-01, -7.6118e-02, -1.8090e-01,  3.0986e-02,  2.0099e-01],\n",
       "          [-1.0188e-02, -8.6229e-02,  1.2823e-01,  2.0819e-01,  2.7228e-01],\n",
       "          [-1.2462e-01,  3.0444e-02,  8.8302e-02,  1.4979e-01,  5.2978e-02],\n",
       "          [-1.0351e-02,  3.2968e-03,  1.2823e-01,  1.2433e-01,  8.1618e-02]],\n",
       "\n",
       "         [[-2.0764e-01, -9.0807e-02, -1.1308e-01, -2.4240e-02,  1.0903e-01],\n",
       "          [-2.7961e-01, -1.9364e-01, -2.1667e-01, -2.7215e-02,  7.1999e-02],\n",
       "          [-3.6868e-01, -7.1323e-02, -1.8089e-01, -4.8077e-02, -2.9516e-02],\n",
       "          [-2.9408e-01, -1.5918e-02,  3.3379e-02, -5.1965e-02,  5.9115e-02],\n",
       "          [-1.5205e-01,  9.5328e-02,  6.2339e-02,  4.9906e-02,  1.7904e-02]],\n",
       "\n",
       "         [[ 3.5091e-02,  2.1305e-01,  2.6727e-01,  2.4751e-01,  3.7737e-01],\n",
       "          [ 1.4994e-01,  2.4238e-01,  2.3916e-01,  2.2945e-01,  2.1685e-01],\n",
       "          [ 3.9028e-02,  3.2901e-02, -1.2806e-02, -1.2693e-01, -1.9777e-01],\n",
       "          [ 1.5215e-02,  3.9685e-02, -2.8262e-02, -2.0420e-01, -1.3177e-01],\n",
       "          [-1.7841e-01, -2.6577e-02,  9.6922e-02,  1.1453e-01,  2.3627e-01]],\n",
       "\n",
       "         [[-2.5082e-01, -8.9300e-02, -4.8001e-02, -1.5038e-01, -1.0594e-01],\n",
       "          [-6.6055e-02, -4.8177e-02, -6.8883e-02, -5.1864e-02, -9.5560e-02],\n",
       "          [ 1.2072e-01,  1.2157e-01,  3.5833e-02, -6.0371e-02, -8.6576e-02],\n",
       "          [ 9.6712e-02,  1.1814e-01,  4.3116e-02, -9.4474e-02,  3.1648e-02],\n",
       "          [ 1.8543e-01,  8.9940e-02, -7.0927e-02, -6.2677e-02,  8.1651e-02]],\n",
       "\n",
       "         [[ 3.4203e-02,  9.7457e-02,  2.1240e-01,  2.6936e-01,  1.1751e-01],\n",
       "          [-4.9069e-03,  8.3272e-03,  9.5635e-02,  2.1389e-01,  1.5222e-01],\n",
       "          [ 2.4647e-02, -3.0144e-02,  1.2904e-01,  1.4234e-01,  1.1356e-01],\n",
       "          [-9.8199e-02, -7.8107e-02,  1.6967e-02,  1.0229e-01, -9.5495e-03],\n",
       "          [-8.9458e-02, -2.8194e-02,  9.8799e-02,  3.5462e-02,  8.9917e-02]]],\n",
       "\n",
       "\n",
       "        [[[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00, -0.0000e+00]]],\n",
       "\n",
       "\n",
       "        [[[-3.0320e-03,  6.2216e-02,  9.3557e-02,  1.0114e-01,  5.0115e-02],\n",
       "          [ 7.0312e-02, -5.2678e-02, -1.7391e-02, -8.6894e-03,  5.3577e-03],\n",
       "          [ 1.2770e-02, -3.1892e-02,  4.9835e-02,  2.1124e-02,  5.1807e-03],\n",
       "          [ 9.9295e-03, -1.0060e-01,  2.6023e-02,  3.7578e-02,  3.8027e-03],\n",
       "          [-6.9958e-02,  4.2977e-02, -4.7678e-02, -1.1829e-02,  5.8378e-02]],\n",
       "\n",
       "         [[-5.5257e-02, -2.3922e-01, -3.5921e-01, -4.2255e-01, -6.2921e-01],\n",
       "          [ 1.2409e-01, -9.6600e-02, -5.5882e-02, -1.8530e-01, -6.0903e-02],\n",
       "          [ 1.9796e-01,  6.2030e-02,  9.4153e-02,  5.6254e-02,  5.5463e-01],\n",
       "          [ 1.4159e-01, -1.6704e-02,  6.8931e-02,  6.6730e-02,  6.8751e-01],\n",
       "          [ 1.3893e-01,  1.5249e-01,  4.1263e-03,  9.3997e-02,  5.4587e-01]],\n",
       "\n",
       "         [[-2.7680e-01, -2.9405e-02, -1.8602e-01, -5.8915e-02, -2.1439e-02],\n",
       "          [-1.0057e-01,  5.6797e-02, -3.6548e-01, -3.0642e-01, -2.3076e-02],\n",
       "          [ 8.7231e-02,  7.9800e-02, -2.1059e-01, -1.7071e-01, -3.1220e-02],\n",
       "          [ 1.2898e-01,  6.9760e-02, -1.1421e-01,  9.7597e-02, -7.7113e-04],\n",
       "          [ 9.4806e-02,  6.1011e-02, -3.5841e-02,  2.5233e-01,  1.4792e-01]],\n",
       "\n",
       "         [[ 1.3933e-01,  2.9424e-01,  3.0809e-01,  3.5567e-01, -4.9006e-02],\n",
       "          [-4.5703e-02,  9.5090e-02,  1.2668e-01,  1.6842e-01,  9.9006e-02],\n",
       "          [-6.5451e-02, -5.3995e-02, -6.3604e-02,  3.9641e-02, -7.9512e-02],\n",
       "          [ 1.4901e-01, -1.3591e-02,  4.3400e-02,  2.1592e-02,  1.0842e-01],\n",
       "          [ 8.8205e-02,  9.1954e-02,  7.7184e-03, -2.1536e-02,  6.8236e-03]],\n",
       "\n",
       "         [[ 4.3268e-02,  4.9329e-02,  1.0790e-01,  4.8046e-02,  6.9622e-02],\n",
       "          [ 1.4233e-01,  4.3726e-02, -7.5633e-02,  2.3246e-02, -5.4493e-02],\n",
       "          [ 6.9242e-02,  3.5919e-02, -1.3150e-01, -3.9797e-02, -1.2687e-01],\n",
       "          [ 2.8571e-02, -9.3596e-02, -9.9860e-02, -1.2213e-01, -1.8505e-01],\n",
       "          [-5.7932e-02, -7.6425e-02, -6.7196e-02, -5.0340e-02, -1.1872e-01]],\n",
       "\n",
       "         [[ 8.4437e-03,  8.0680e-03,  6.1450e-02,  9.9150e-02,  6.4177e-02],\n",
       "          [ 2.7052e-02,  2.1411e-02,  4.0456e-02,  1.7061e-01,  1.1688e-01],\n",
       "          [ 5.1299e-02, -5.3776e-02,  9.6704e-02,  1.5999e-01,  1.4347e-01],\n",
       "          [ 1.3904e-01,  3.4033e-02,  7.1747e-02,  1.9670e-01,  1.1705e-01],\n",
       "          [ 1.2442e-01,  1.1442e-02,  9.4426e-02,  1.3482e-01,  1.1477e-01]]],\n",
       "\n",
       "\n",
       "        [[[-0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00]],\n",
       "\n",
       "         [[ 0.0000e+00,  0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[-0.0000e+00, -0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [-0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[ 0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00,  0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00, -0.0000e+00, -0.0000e+00]],\n",
       "\n",
       "         [[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "          [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00]]]],\n",
       "       device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model.conv3.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 2]])"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ixs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([ 1.0643e-01, -4.4014e-38, -7.0284e-02, -7.1342e-40], device='cuda:0',\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model.conv3.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([ 0.1064, -0.0703], device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.conv3.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.1722,  0.0148],\n",
       "        [-0.2179,  0.3632],\n",
       "        [ 0.0464, -0.1530],\n",
       "        [ 0.4846, -0.4002],\n",
       "        [ 0.5205, -0.3365],\n",
       "        [-0.3677,  0.0714]], device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.fc1.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.1722,  0.0148,  0.0961, -0.4640],\n",
       "        [-0.2179,  0.3632, -0.0145, -0.2176],\n",
       "        [ 0.0464, -0.1530,  0.6069, -0.2542],\n",
       "        [ 0.4846, -0.4002, -0.1324, -0.3261],\n",
       "        [ 0.5205, -0.3365,  0.2381,  0.0093],\n",
       "        [-0.3677,  0.0714,  0.5205,  0.7617]], device='cuda:0',\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model.fc1.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8577752, tensor(0.7179)]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = data.one_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[12.0570]],\n",
      "\n",
      "         [[ 0.5953]],\n",
      "\n",
      "         [[ 1.2274]],\n",
      "\n",
      "         [[ 0.0000]]]], device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "output = learn.model(x[0][None, :].cuda())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([-0.0999, -0.0308,  1.2274, -0.0048], device='cuda:0',\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.3803, -0.5426, -0.5231,  0.0725],\n",
       "        [-0.0651, -0.2809, -0.3671, -0.3667],\n",
       "        [ 0.3661,  0.1138, -0.7207,  0.4257],\n",
       "        [ 0.2243, -0.1901,  1.0422,  0.4037],\n",
       "        [-0.3652,  0.4749,  0.3714,  0.3622],\n",
       "        [ 0.0806, -0.2849,  1.0241, -0.1796]], device='cuda:0',\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.fc1.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "    def delete_fc_weights(layer, ixs):\n",
    "        \n",
    "        weights = layer.weight.data\n",
    "        ixs = ixs.cuda()\n",
    "        \n",
    "        #biases = layer.bias.data\n",
    "        weights_keep = weights.index_select(1, ixs[0]).data\n",
    "        \n",
    "        layer.in_features = weights_keep.shape[1]\n",
    "        layer.weight = nn.Parameter(weights_keep)\n",
    "    \n",
    "\n",
    "        return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1]])"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ixs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = delete_fc_weights(new_model.fc1, ixs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.3803, -0.5426],\n",
       "        [-0.0651, -0.2809],\n",
       "        [ 0.3661,  0.1138],\n",
       "        [ 0.2243, -0.1901],\n",
       "        [-0.3652,  0.4749],\n",
       "        [ 0.0806, -0.2849]], device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}