DeepMutualLearning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "V100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "2dD8l0gyiltA"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision.models import resnet34, wide_resnet50_2\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
        "\n",
        "batch_size = 4\n",
        "\n",
        "trainset_cifar10 = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
        "                                        download=True, transform=transform)\n",
        "trainloader_cifar10 = torch.utils.data.DataLoader(trainset_cifar10, batch_size=batch_size,\n",
        "                                          shuffle=True, num_workers=2)\n",
        "\n",
        "testset_cifar10 = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
        "                                       download=True, transform=transform)\n",
        "testloader_cifar10 = torch.utils.data.DataLoader(testset_cifar10, batch_size=batch_size,\n",
        "                                         shuffle=False, num_workers=2)\n",
        "\n",
        "classes = ('plane', 'car', 'bird', 'cat',\n",
        "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3q2Fp-3Ml7L-",
        "outputId": "cb6bc6e3-9312-4b64-c4a4-97e837937f20"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 170498071/170498071 [00:05<00:00, 28985345.79it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "# functions to show an image\n",
        "\n",
        "\n",
        "def imshow(img):\n",
        "    img = img / 2 + 0.5     # unnormalize\n",
        "    npimg = img.numpy()\n",
        "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "# get some random training images\n",
        "dataiter = iter(trainloader_cifar10)\n",
        "images, labels = next(dataiter)\n",
        "\n",
        "# show images\n",
        "imshow(torchvision.utils.make_grid(images))\n",
        "# print labels\n",
        "print(' '.join(f'{classes[labels[j]]:5s}' for j in range(batch_size)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "id": "-z5pdHtZl-C-",
        "outputId": "62311cf9-96eb-44bd-a584-cd013a52afde"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAACwCAYAAACviAzDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSZElEQVR4nO29eZAd1Xn//XTffZl77+yjkTTaQYAQxmKTcWxiy8bYwTiQxPaPxHipuJxIjoGq2MaOnYoTIip5K17yYlxJOdipmOCQn8FbbMcWGIIjsQgJEEIb2kYazT5337vP+4df3/N8HzGDBOIKoedTNVXd9/TtPn3O6XN7zvdZHGOMIUVRFEVRlDbhnu4KKIqiKIpydqEvH4qiKIqitBV9+VAURVEUpa3oy4eiKIqiKG1FXz4URVEURWkr+vKhKIqiKEpb0ZcPRVEURVHair58KIqiKIrSVvTlQ1EURVGUtqIvH4qiKIqitJVX7eXjzjvvpMWLF1M0GqXLL7+cHn/88VfrUoqiKIqinEE4r0Zul+9+97v0oQ99iL7xjW/Q5ZdfTl/5ylfovvvuo927d1NfX9+c3/V9n0ZGRqijo4McxznVVVMURVEU5VXAGEOFQoEGBwfJdV9ibcO8Clx22WVm/fr1rX3P88zg4KDZuHHjS353eHjYEJH+6Z/+6Z/+6Z/+nYF/w8PDL/lbH6RTTL1ep61bt9Jtt93W+sx1XVq3bh1t3rz5uONrtRrVarXWvvn/F2JuueUWikQip7p6iqIoiqK8CtRqNfryl79MHR0dL3nsKX/5mJycJM/zqL+/Hz7v7++nXbt2HXf8xo0b6a/+6q+O+zwSiejLh6IoiqKcYZyIycRp93a57bbbKJfLtf6Gh4dPd5UURVEURXkVOeUrHz09PRQIBGhsbAw+Hxsbo4GBgeOO1xUORVEURTm7OOUrH+FwmNasWUObNm1qfeb7Pm3atInWrl17qi+nKIqiKMoZxilf+SAiuvXWW+mmm26iSy65hC677DL6yle+QqVSiT7ykY+84nM/u+XfYH/VOee2tqv5JpT93x/9Cva/9v/8fms7TGUo2/H0kdZ2zYlDWb6CdTh35fzW9rbHnoSyJYutrct7f+8tUBYL2hO9cKgIZT/7yXbYz06VWttbth2DsvPOtfX74z+6CMrS3SnYT8btsaGOXigzrtXlChNPQ9mDj1Vh/++/bI2Fg01s5z/+yBta2+uuXg5lpWkf9qezdn/70RU0G9kxvOemh9f0fXseqS/WyrZtG5UslEUCeJ6OZKi1HY6GoWxqJt/aLpUbeI2Ggf2xKXts08N3+sGFS1vbF6zC/ioUbP0OHT4IZdJVLRy29atXsT7lYqG17dfyUNaVjsJ+lJ2nWMLBXSza7/o+thURtnNHup9t47gbPmrl0yvf/l6ai/v+7//b2jY+jhfDIgHIfp5LV5YlLusuVxS64gM3YLcDAXfWYx0Hx4Axtu6+kWWiQj67pi8KWRMYbA4iD4+NsDaIBQJQFmZ1D4t/M8OiDcLsvoIk+sCzY61JOCacIJ542btvptlYPZRsbe8bnYSyOuHqt+PaMZqdHoWyiLHjOT3QA2Ue4XPRnLHPRThYh7Ike96rFSyr+SHYj6Qyre14An8fRo/aVf5KzYMyYvNvMTcDRdljI7D/9LZtre3udAbKzjv3nNZ2vVaCskgc247XNZjE59Jx7X0Vp7EPxo4ehv38jK1vXw+GyDjnDZfQK+VVefl4//vfTxMTE/TFL36RRkdH6Q1veAP99Kc/Pc4IVVEURVGUs49X5eWDiGjDhg20YcOGV+v0iqIoiqKcoZx2bxdFURRFUc4uXrWVj1eL33snak0jI7nW9k+2PA9lwSje3vanX2htL+pLQNnhY1brfmbfUSiLp5KwPzLNgqL5GExl8hn7XdfFoGqLF6Rb276LuuHIONqAjLH6HJnCsu5Jq/EVJ9B25amtB2E/FLN175u/CMrGR6db24WJI1A2iZckClgt0xBqy089be0zjk2grpkQnkzjY/bE8y+a3eaDPNSdA+I9Ocj07HIpC2XF/HhrOyYcqZId8Tn2hfbv2vFTKuWgrFLF+lXKdkzEk2koS6Xs/tDQYijjnl6pVPesZURE0ajVustltNU4MnygtX34BXwOSkLPjoTtPUci2B7Vij1vuV6DsnAYdfAge7y6urDupSp+dy5cbtggbBHAwsGRhhOS2ct9h11D/MvlCFsJn9k/GIMavsu+fJzFCb+8sOMwwlbD8HJxrMPtQURlHYNXbbLvVqWNjs/qKm1XxMwfctixon3A9sgIO6QIjom5aATs+PVcHNu5HI7nUjnb2q6UcDLqito+6XbR5iMpniE/FGttV6dxjtu10z4nRtxXIo3noYKtQzQWg6KpaTtXi2mBXM8+e7kZnEMiIXz2liy182Fvt5hDOq3tRsDDtuO2YEREJVaH/OQE1sexY71Zx3khFMTzDAxY28Zw8NS/KujKh6IoiqIobUVfPhRFURRFaStnnOxy6eXo8vPQf1vZYXQal/xXDnXCfnbMLnuFGrg+FmJL2vEULo8NH0EXJCdgl726e/EaYyOHWtv/8/B2KNu/0Hr7JJK4dDcxgUuLUwXrMhaP4FLryKi9z8cex6XEWApdq8YnravZwUOPQVk5b5fGgy66Yx7N4RJujK2upjrw2Jlpu3zXI0L6N3102fXEUt9sRMRSYlO49xZL1t1sahxd1lyyS7iBKMprbgDbstGw7ewJL7l6zZaVirgsnC2IZeKK/XI4in3Q3WVdnLu70d25M5Ox3wvhcmpS5EdwmVtlo4njN5m08trM5DiU5SZQRqyEbR+4wl01HLF9K5ffq1W851qdj1k8z8IFi+lECTF3Tek9y+UJI2UVqbLAl7GQKwdBMQYCQnaZK8+3yxpFHsfrd1xdxY0ZLiFJOYkd6wg5YC73Yk9UqNpkbsqEg1uoMOB27wq/3Ahbcg8G8NmPRPA5nYv9R60EcHgM3TyNgz9FDVb3hnDDbTIpw9RwTLqE83Guao/1PSEbxq1kMzE5DWUFD91ZGxX7THkVdGV3wnYud+I432QStn16ezNQRuI3KJmw99mVwN+HMJMmY8K1diKLv3vTRfubmCvg70o8aNsgEMT28OpCtgsySa+C8/ipQFc+FEVRFEVpK/ryoSiKoihKW9GXD0VRFEVR2soZZ/MRTKM+++wuG9rWb6I9wbxudGVKJKzGVaqjBuow24TFA6ipBRroajvQYzX93gWDUJabZi6PR1EbfO6wtR3pzqBWGgjifTks/nFXGo8t5u19HhYuukMh7NI6c5d0AniedMre50wWNc58AV14B3ut61ciinXtiFjbiFQSdUTXwXZOd5xYEsFICOvqCxuHPHNbK+exnUMB2z4FoXXLeNXcnVVE9qbxMatR5wvYPvU6HtwEfR3f6Q1zj4xGUDNPs7Dk8QSWJRI47rhtggzhXija+mU60a5kcgxtPvi9ZERY9M5O62JYLst7Rt23VLLt3migXrx0mXUbnKnOrReH2diXFg0+j4s+lzGGwBEhwgOu3Y+IZy0iXIh9ZmfiedKwg9l8yNqyLmkKgxnflSHLuc0HCZjtiI+Fjjivw9zefRmm3dg+8WXIdGk7wk7rhoTNB7MxSEaF+7ewzZrLouvwMWvnka+grUZBPF9dnda2rzONdhyJsp1H/Um0ectXMLz62IS11YgGsJ/DEWuf0dmDZeUyzn9OmD2bDo6fRsDOuT2DC6FscPEye/0gzkVTR/bB/sQMszvx8dk3Tdtf9QTWNVvE34Am+1nv6kaXYcOOzQt7kHFh95LssO0j7UNOBbryoSiKoihKW9GXD0VRFEVR2oq+fCiKoiiK0lbOOJsPdwa1p5mc1QplOu60CKXNw2UfPoyxEJJxq+lVK6hfN4X+V8pO2e91Ynjfo0etjcXEJH6Pkyuh/heNiPDHYVvXjgTqqlX23VwZv1cRdaWgvRcZFp2/ehaFBluto37c1Z1pbSfCWPd53bbdK01xHyKGQf0lQ2T/muPCGYgw16WC9W33mljms4AdTeGvb4TdQCxqNWJfhK6u1ayCLbV2GQQjzGIjRIRdR7Vmx4S0jUh0WG03HMb+iYrY8A4zDiiV8TyhkH0ugiJMsiNiKNRYbISa6Pcoi9vginsMuDI4hK1DTcQBwFDNc9t8uEF7XwERH4P3lyNDlou+5H0k655gMVTiMZwXQiJ2hc/CkteqaEPgsPFrHBlL25b54j58YTvSYDZnDRFfocFSCzgujomgiIvuMBsmT6S7N1CGda2Lsc73Y6KbgxF7zVAU518+7ojmtvmoNFgsHJE6ICbi/4wcPtjaToqzrh2wYzYthtbIgYOwz0znKCDsUzyWXr4rgWOiK4ptGc/YcsfFGBw11kdpEcujyeYpU8HYJoFjaPMRZHZsjTjabdV9W/dcDm862NsF+6m4tc+TUVhCzGYnlzsEZd1d+FvmsmAwkzNoD3Iq0JUPRVEURVHair58KIqiKIrSVs442eXZLbtgf2LaLqvHhRtYuYJL7hNjLPtgUSwTs+XMhpAc0kxyICJKdVk3sNzEFJSFjXVfWi5cdnt6rMtYKCiknQYuLVZqdonSF8vmlZRdTBMLv9Qp3CzHZ+yyn3QXTTO3WBl6OJYQy7ssBHalgtJOrWaPrXtSDsAaTk5YCaL3XJqVWgOzos7kcMkym7OymeznAFsuTAgpI+Bg3V1jy+tiabzM5JKmWKY2x7nTsvapYd2PsPD8h4dxqfO8Vee3tiNRXCSNiKzMLlsmzuUwW2V2yrZHKY/tUasJWYq1bUO4mXo85LOQNXgYdCKiMFvUbcjQ62J/TpgbqpTbQkxqEZengJBWuNt0WIRM58vvQUe41jaFnMMz1wqZw2Ou20aomEHm5i7lCOkFW2eyS6WMz37J2LnJl+HVhfRlfCYLGZSI+JPXlNcXUqXH5jynjteIMfkmLmRTX0pPcxCLWdfNbiEVVIsF2D+8a09ru8PHJf81F1l31sB0FspKMqt23PZJQISNL7HmauI0Tpk0SiupOpMjhftzqMO6qzeF++zoiH0u40KKi5XwOY2wqaksfoOOFWx/RftQHumMYl1zNTsHJ8XzfWzEZiCfmcniefrmwX4hb2Wg/j78XTkV6MqHoiiKoihtRV8+FEVRFEVpK/ryoSiKoihKWznjbD6eeOoA7B9lNh8dIk19yBWhtdl+SOjpHnPTDYsQ5dGONOyneq02NnkQbVCW91ld89zli6HMMBX2yOhhKEtkMBWz8a1uV2yKEMYstLYTQJ03Ece6ujNWew+KY7lLZkcHXqMh/O2KzJWynEc9P8YEdZkGPhDAtmw2Tiw1c1No0nmmPxIRBVgK8FgM+z3LUkw7vgznLlwePSu0NsQ1edzroAjJ7Xuo+/K29JsixPOo1VmfeWY7lF166WWt7f7+5VAmIswTN6eZnBiDsme2b2ttj46MYN1cYUjBw5kLIwuX7cto5tKOgbt51oTdTaXMQjeH5w6p7zAXVVkfHqo+GsaxlI5jv6ej1mXWaQq30wp3mxY2MBHcr/rWJsYXNg41Nnybden2ajssHMB7joRw/PBd10UbIY/ZF9WELZhnpH2KrYMn4rR7bKx70vVXuK7zsV8vYdt5IZbC3sd5IR1Dm4+5gnDP7xlobcf7Mex3Wczd5/Ta8hU+zhmppp1/mg4+azFh0xVnYdF75uHcdPioHbPTObznnh6cjyOG2aTU0D6lWbPpC2LCzX1xxLZPo4H9U3BFX7J5dCSLKRECFds/84fQXsYVaQ8KNdtH0r14gqWMCKUzUBbN4HnTSdt25yxBe5DZg0icOLryoSiKoihKW9GXD0VRFEVR2soZJ7sUasLttGqXBDtSuNQZF9JKmbkgBcRStN+wS2CxDGYUjHbhklOjbJf1gwaX4BJpGwmvvyMDZWM5u1QejAi3yrjIpOvaclPGpc1k1Na9ITIs1sVyc4K3gXAbDATseVzhmigVqwBz/0sIlzW+MpzPY6bEWgiXUwvFE3PBTKdRPpo/fz4ewJfGDd4zz8ZaElku603R7751vw4GsSzMos5GxJJ/ZyoD+03Ptt+MaAOPRbDc8fTTULbjmWda26svRN/jkBijlao97+NbtkDZtq1PtLaDwjU7JSL9msbs7pENNn5icYz86Ql37CbTIOo1vOd81rpGR/tE3x0H7z8xJTEZJiAkmQ7hRt0VZeOyggM4nLDPV1C4H9aFO3iWuQk7Itoxl9uawg02xKQLR7hb+2LcuUF7n0EhicQhiqiITCqe7wqrTxWLyGPt5YhMtUa4KfN7qQu/3FrWPkN1EfG1S8i1S2h2UkkW1bWBruKhBvq6Lkrbhf2oiK65d9iOrVAYx3ZFRuFl840RfRAJ2rZ1PZS3jh1F2cNnkYnDIpJ2mEWAjcbwGgk27mp1HBMxMeeXWHbutOiDJJMYIxN7oMytHIP9eTE7d9bKONYvXGCf6RkR/1S6jg/0WPda+Xt1KtCVD0VRFEVR2oq+fCiKoiiK0lZO+uXjkUceoWuvvZYGBwfJcRx64IEHoNwYQ1/84hdp3rx5FIvFaN26dbR3795TVV9FURRFUc5wTtrmo1Qq0UUXXUQf/ehH6frrrz+u/O/+7u/oa1/7Gn3729+mJUuW0Be+8AW6+uqraefOnRSNRl/kjCfH/iMYPnew074/LcyIbJ4GNbYE0+YqPgqkbsjqkd09GEo2M7AU9g/vs3p7LITnCbhWx8uXUYvzm9Y+xJGhkL2y2LfaYTSI2iBkHjUyPDa6XVWr1v5Buu2ViywrqY+2GW4A27I3w7KCpvE8HV1WYywy3ZKIKCEys5araE8zG2GhMQZEds9Ewp6n2ayIMusmVyjieKkJLdX3bb/HhHtxZ5cNmzw40A9lAwOLYL/J3CyffmYHlB0cHmb1QduI55591tatci2UxeLoGrhr5/Ot7R/c/z0o46638/sHoay7KwP7ftOOmVIebZaMP4fbq9gPMTuhhnieigWr07+UzYfjzm6DwrPT8rD5REQxEW+9k9voCLfgJsuoOlPEMTqTxTFRYbt1YUjhMu2/I4jzS4S56Dsks9GKuY+FePeELUCgztz+RVx2T9jr+DU7j/Astr++BHObPi4suthndieO+J+0zKaqEZF9dqaG7TOXzYcbYykjjqFNhXcQn5l43bqLh+I4F0yxNokm+qAsuRjnbn/UhmYo5nGO7UrbeSKdwmetUBL5eY2db1xhHGHYXJ7swH6OROyxHWm0T/GawlWbDQMzjs8lDwXhV3EOiSexDxb12usUCG2fhs5d3dr+xWa0HZmpYjiDQt3eSyazAMpIhD54OZz0y8c111xD11xzzYuWGWPoK1/5Cv3FX/wFXXfddURE9K//+q/U399PDzzwAH3gAx94ZbVVFEVRFOWM55TafBw4cIBGR0dp3bp1rc/S6TRdfvnltHnz5hf9Tq1Wo3w+D3+KoiiKorx+OaUvH6Ojo0RE1N+PS9T9/f2tMsnGjRspnU63/hYuXPiixymKoiiK8vrgtMf5uO222+jWW29t7efz+TlfQK5cjbYJkwV7C5kOtC/wA6hrVlmI5VgMtblozGqw0c4MlkUw1G4/s39oVvCasXgn+55IQc5Cjae7UGPMdIqYCr7VCrNZ1PjC3D6lEzXPqNChAyxGty+7m4Vxnihi2UQBtcI0010vXIK6apPFsaim8J4HB7F+4VDWXp5OnKaHumaYhQ0W0fDBNsAX79eeh/s+a+ewCF3d1WPDDZ97LsbgSMQwDkmT+czLmCk1ppNnRQwQ37PtXBWxIWoVtFv44Q9+0Nreuwf12sF+G4tmcBDj0nRlsK6Nuq1frYz2Mi6LkyBttOo1kSa+YbXugAhZ7pAMVT87hrW7ESnaeZe4Iq5GVNh8JFgsi6QI619lMTjyNezn4iS2e7Zsz1Otog3VvG7bz4vm4VzUlbLtxW2/fl153C+y+CFjMzi2J2ZYOPM6tketIcLG120fBCPChoC1T4PQhsGI+CH8YfQ8LPO4DYx4noqVE3+Kd+2xdh7pCYzz0StsHGpVuwJe9UUsj5i1vwh2YXr5SBTtOngMjr4g2psl2MTR8NAGL5LphP1G0/atEfY8TRb+PZnBMZFK2+9lC5iCIJXEEPM1ZufXF8Rnb2I029ouVfE5LE+jLdZIxdqYxQfR3mrP89aW5sDeQ1AWGzwH9qtszGZr+DyfiheHU7ryMTDw69j9Y2OYd2JsbKxVJolEIpRKpeBPURRFUZTXL6f05WPJkiU0MDBAmzZtan2Wz+fpscceo7Vr157KSymKoiiKcoZy0qsnxWKR9u3b19o/cOAAbd++nbq6umhoaIhuvvlm+pu/+RtasWJFy9V2cHCQ3ve+952SCv+fj18F+zu32iWmsEgDms3hEly1zly9fFxGMmw/1IHLc/EELsuuzNilLDeAS9rhiF12S4ism42SvUZNZD6NxXEpLcIzgQpXxIMHWehqcY3OXsxMGIvZJcKmcNFqsqXXn2xB17eJHLpSpZn0tGQR2vQcPWJdivu68PrnLsVj83m7RIn5iZFmHdunJpa/PRZm2mviknKTudN6YvW/Jvqdh2aP+ric6vm2T4plXO6eHBftNWWXjadzYnk1Y5eGaw28fiDA+la4qw4fOgj7O56xodmDImtqL3NnjQg35WpNtA9LJVAV4zAYZS6XQtYwwt3ZZ/KWI1PgymX9OWg07XnCQirtStp7WdqHy9TL5+HY6mapBKp5DMntNG19Mmlc0k6WcaxP8n2RCbojZSWAxQvx+vO6uTsm3kcojNcsMv/VznFs13TCjrvxHF5/uiBc9Iv2u2URMt2w9AkyG60RY81jz4UvpS8e/l5IXyejnY6P23nLY5mniYiWdaC0Hc/Yvh4/moWy/UfsPLZ9D8o3qQSO/eWunTei3UKWYvtyrOfFM1NnykamB38fYlEroddklt1uW9ZI4D1Oib4ssnQBgU48NlC2v0ERIZHXRJbxAHPHNgns521bt7e2x2dQ6rpwFUrk6T47vuPC3Vl4h78sTvrl48knn6Tf/u3fbu3/xl7jpptuom9961v06U9/mkqlEn384x+nbDZLb37zm+mnP/3pKYnxoSiKoijKmc9Jv3xcddVVYCAmcRyHvvSlL9GXvvSlV1QxRVEURVFen2huF0VRFEVR2sppd7U9WbrO+x3YvzT0SGvbES5ideFSx+X+Wh1tCFzmahWKoMeNK9I2J0PWJckE0Y4iwFLIBxzU5U3NanPTQst9etcRrGvNaqKD81Drbrq2fj39GShbuUq4wZYN28b6FCrWJiYeQduRcAR1xEjQ6pEJ4UK8ZLG1e/GFS1Z3BvXsaJR3As2OWF2rV/HgBnNLDQTR1Yx75XrS9MAVWjy7LyeIdjfFotV9Z3I4XqiJJz5yyNqAjOVwTHT3Wy21rxddAzMZq+2WSxhgb3Ic9ey+XjsO5i8YgrLxSaun53LjUNbfg663U5PWG61cx2suYrp0rSnsY4TNh8NCmLs+9kGzgeN7LvhzGY5iH5yz1KY2WL0UQ9qnhC1UdcamZc8W0N7r2Biz4wiifj0vgxr+AHOXTAqtu78n09ruFWHr68yleiKH7RoVofsjYXuN7h7hCRi2Y3umgLZFsQi2cy+zc3Oi+Aw3yR4rpkIqCpuq6ZJtr7x81thDFCT8XuAkbHv6euy81SDsn3oHts9F86ytxMWX4JgI77B2fs/+AvOGmRz+pGXY85bLom1PdsLOsStXYwqNeDeOCXJtW4Z7MExCitlnhCP4jJR825a9SzBE+b6t6C7vdlr33rDBfs6wn+rJUbSXKdeF+z6zHYklsH8iSbu/IIi/c/3dwiWf2cF0dmJ7jM2gTdXLQVc+FEVRFEVpK/ryoSiKoihKW9GXD0VRFEVR2soZZ/MRFvENgh02roRMKd3hoI9zs251RieIftSGpZ53RQp7J5qBfa9mj62XRJpx7mcutGWeOjzmo44ZiqJdyXjWHrtrywtQNn+xDT8fw6+R2xCxTQrMdkMEvQgyDd8XIYNjUdT4OhP2u2kRmj7JYkw0yqgJx5Pi2CSzSZnD5qPRxL6TrtoOS2M9k5uEskKB+a+LMREUaeGDLAx3TDRmhfn6T0zgNRbMQzuKIabnzjyPWm6YdfU73vk2KFt57srW9oObHoSydCYD+1dd9fbWthv8FZQ9+cRjre2ebvTXr1UwZkGZ9ZEw1QBTm4wIMe2JENR1pmfLFPKOaOe5MLyrsdupI2F16UwHatTBGmrdNRbSPRjC5yuQzNgyUbVu8QwtYKkPejrxmh67r6KIm1OusPT2HhpZlIUNSNnPtrZF6BlqMrupTAj7LigO7mY2S1Gh73uOHdsVMdWXDD5PkyU75+0bxgjVVRY/xD9uwNAJU8ragyM9aLM03EQ7pcQhGzvosstXQNmVb7PzTcFB+4uZF7DufWw+DImQ+7mcvZeqSLuw4sLlsP/UXjsH7zk6BWXnpexvUKCO1zh40NrsvHkehi+Xz3ed2VjNjGJ7lIt2TvMM2p9Fk2LuZnZksST+li1baO06qmjWR14e2y7EQtd3dmP8prH9h+mVoisfiqIoiqK0FX35UBRFURSlrZxxsovTQNcqP2uXwwPCTc8TbmHZUbuUVxVL0SEWijjTja5vEQdd2EpT1tWrnEf3rd6IlSuMCHfsNViIXOGSdf7KQdgvl2xds/uGoewNPXbJdF6/kAqyIixwlYWND+N9FIp2SVKGg05GsX7xmF3qC7l4X4mYrXsoicuytQIuHxr/xIZcQ7hqBoP4vSBzCT14GAO1O8wtrq8H3ZSz+Szsc1fbiAiBnZ22fVvOo5zVFO6IF5x/XmvbEy67569e1dq+7n3XQtmD/22llvv+4z+hrLsfpZ0bfu8PWttXX/MuKDsybFMeJGLYz1OjuORvfNvXPHMvEVE6ZaWWTAbLGg2855maXQo2vnDpi4isrnPQrNnveiE8T5HJFaUiyhzzOlDWTPZbSS8eFuMsbM9TKWFY6aiQNjIsuWWqA/uSRaancAKvkUqzcOY1HL/VIl6Tu+XWhNtrmM1b4R68fs3De64x9/1IBJfjHRYiYKqM49cr4TWdov1uoJKFshALqS7drX06cXlt/hKbGTozX0iDMyOwv2/H1tZ2bjM+34sX27DfV73pjVDWXJ3F8/zvw63tRgHHb7li635sEufxpAhZXmfzhG/wPFzhiwmZeegc68JrROqAVBpdW6OddvzW6kI+H7PzaF8UTQYa0sWaK/9CBjczdn9qCueFSn0f7Pd22d/BRBTH3alAVz4URVEURWkr+vKhKIqiKEpb0ZcPRVEURVHayhln8+H56ALKw2yTsEWQLrORqHUd8nzU1BxjNdhmVfiPNdG1qlmyemCSufAREbmOtbFoVDG1erXAXRNR254cx2PHRq3bUyyCdh2psNXfQg5qccUGapUhFiK3aYSrJNOW16xEDTYnbBwGujOt7a4+tIlplmyo3XAc9cegI1PYn1g4ZulaW6/gfQYcq8FK7TSVsdppRxLdRUdGDsF+ibnTloUdUDxux1pC1Mdr4Bg5dsxqsm97+zuh7F2/8+7WtmyPQwcOtraLBbRpKNUw5P7IqLVvev/7r4eybU9uaW0/9cTjUJYQOnQoZMdEKo2upPyeHfG/STiMz5PLnjfjOuLYE9eIHea+2aijry0PiV0Q9lUDcezbjoQde/Eg9g8fltU62lFkxPjp77fzRDyObWfYc9sQecWLOfsMz0ziWPJF+0Q7rIuoL2w+ilUbPjsn8gMU6zh+uHtmr3DdpKCt+9gkhuo/PI5tOTxp617xsd99mFfxPk7c4oMoM2TnmEwX9l1NhLF32XOb3fYwlP3ikR2t7d4utAdZvgjnpkUXXtLajtSwT/buPNjanq5hO+eKuB8OMTsLF5/TylS2td2TwrFUYy20fy+GTOhIYV1Hjtg5PxXHPuhfbO2vxvdhaPOieGbizOarksVQ7EePWNujQ3lsj2ZtFPbrGVtfrzZHXISXia58KIqiKIrSVvTlQ1EURVGUtnLGyS5OADNbhuK7WtsBsQboOMJdtNsu5cUyuNzsM9dJry6WoxrCDSvJZJAQuj0995xdBpyeRPe6wrRdAvMdvEa+LGSYGds15SqWxTvsfQUj2IWxlIgAy5bYixVcnhtgrl2LluB5jGiDEI/+KSIylst2GTIYE1JXBrPsJrvZkmGWZiUkIlTOFeE0kUApznVtHRJCBhoc6If98Slbn2QKj100ZMdaTye67Pb34n31Mynqyt9eB2VdrA0e+vlPoCzLry9c5io+jt/JyWxrOxTEY1essK6+R4YPQlk6gVEgC8yNsN7AMcFdQJsiq20kNJf7LD58rnDJnJMAlwbxPDw77eQMugb2d2B9Uiwzc0qUJZn7rBPEJf9IAueCALvPpnBBN0wGkWXE7tmI/qmL5iiyZh8Tbp2jTPEsi0jIefFc1j174pzIvJxlLsX7j86IMpER2Nj6msDsfRd28HuBkwhxWmMRp6tNXMYPhnD+ma7Ye3l07zEo2/OUnWMXJrCfdz+Lxy7ut329bCG6rvcsWNbaDlRwjl84H5/3mZzth4kJnNeP7LVyxfQInsdh4Y1FVxL1opw+ssf+li3uxXZOuLbfIxGU6dw6Hruwy0o0RyrYHmUWpbjpigq5OMc2mrZv9+5DyehUoCsfiqIoiqK0FX35UBRFURSlrejLh6IoiqIobeWMs/lwhWGHw8LeOiLjo/FRzw6w2zXitctlOqcbFmHRxTsaz547cwzdk/Y8+bz9nocaNc9IWRRuuEWDtiNjE/aaPQtQo160hB0rsodG0nhsg2mrQYPtEWEhy40I5+sKbddlIebrol0jzB5Dfo/ENaWL8Wy4IgNlKISueD6zRzgugyizQekUrm/xHrTViMRsWzoB1DwH51uNeO3aK6EsnUFNeGCetQ/pn4+h8usl29dV4cIcZZmP+7uxbrmCsMco2b6cmcbxs2y5zZhZrmBI+0IONeqZCTsup6fQjbzK2k66kgaEKzt3d6Yw2t2EozgO54K77HL7DyKiOrMpqMv0r0G0TQiE7H44Ip5h7gIvxm+1gq6TLnMrNMIGhdvBlEtoY5HL2b6dmkY7joLMLMz09OkillWJ2wnguK+LcNnTzO2/OI12HRMs1UKhim3V9NFWgmerdYRRV4hl4w4Il+HASdj2dMStjUNIPLP7d+6C/f/6/g9b27uewyzRxELXF0pY18ksts+hcds+Ow+hu3F/xo6RZctEqHOc1qmz07oJjxm85tSMbffJSXSD7ehgc7XIPhsS9nrnDFhbDW8Gs8aWWArw7j5Me+BEcDwXWR1yImx8k1WhLvrOFakvUmyOq1TU1VZRFEVRlDMcfflQFEVRFKWt6MuHoiiKoiht5Yyz+fCaWdj3uVzaRO3Ua+B+OGY1fU+kAPc9q+NJu4SaCMtbY5rjOAuDTkQ0kLLnaQof+KMTzK6EUCPfJ0LmHpixx6590zlQFmbad72Jel/QEym4i1bzDEREyGtmd9LMoV7M9XMiIjJ2qLhh9E93HBanoYK2CIEa1s8/Qe2wIWJMdIjQ0dWS1Z5DQbyvZsN+NxzG9kgmsd07u6ydRaa7B8oWL13e2l55/vlQViyi7cb+3VaX7u7D84RS9pqdQwuhzNlix2RUxIZwUIameMLeS9PHdkylbSyPnl68foeI85HpYGkGPOyffN7ag4RE29UqeM/8f5dwBG2WYnF7TbSMOJ4QsyOIi3gPiai1TYhF0U4hHBbp3dlzIUNwBJl9Cn/WiYiaTbQTaLDnv+mhjQOPlTNTwPaYyNqxPy3sQapNrFCFhcTO1/D6TdauvpiLZkp4zbFpa69SFjFbmp5ty7CwoQrK+BzMVssRdlpBlgo+6IrzyOBKc9BgYcgP70ebhi2PPAr7+5/d2dr2RSoDPt9Mizm+IOKHHKvZfl/ZgTF++rvtA3buxfh8n7cUbZZcZltXyOEcEopb27B6TfRz0fZPsgfnzUXnZGCfRuxvwMwErgvwOCyRMF6jM43P9/gRa8eVE/ZETTbHZKs47kpltP/KTFt7kVUiltJUBX8vXg668qEoiqIoSls5qZePjRs30qWXXkodHR3U19dH73vf+2j37t1wTLVapfXr11N3dzclk0m64YYbaGxsbJYzKoqiKIpytnFSssvDDz9M69evp0svvZSazSZ97nOfo3e+8520c+dOSiR+vex6yy230I9//GO67777KJ1O04YNG+j666+nX/3qV6ekwrXyFtg/cvRga9sr45JbVCwbx+M2jLIrXT7ZUqwbxKVEzxfuZWzpaiKPS2C5GXveQg3lgG07bVbS/k6x7CmucfXaVa3tq9aeC2UNtvRbL6EbZc0XS8hNe95kFy7H++ByKGQo4XYVYGGdA2Lp1WOueK5c7xauimROzDXPFW6daRHevJdJG5EoLmc2WF17uvF7XV3opsZdenv6MfzyinNWtrbDMZQVgmIZe/9z1lXw8V/9D5Sds/KC1rYRYfR7IrY+6R6UFeoBlAfm99jl1WJ+EsqSHVa+WbYUZbpqGSWActE+J01fukbb/ksIiWrvbnSH9Im7tqJk5AZlLOnZiTKXv5AYLj5bVm/W8T4CYsk/xGSYcBSfPYdnUxbPmi/Oy0PO58ooQYxl7bHjBVzSni7Z/ZyQG4sVvEaFSbk1kZXUZxKRI8ZZVZzXYVmtwyL8fZi5LQvvUPJd+Zyy9nHw2eMemUFZdhLh1Yd37W1tjx4chrLsGLqHGyaFyRDuPhujQhUjT/w7XWM3vnccZYXOTtt2B4+iS2pfCk880GOfr4ULUeYIMJfv6RmcfxMsS3TVz0KZ6+OYKDIpQ3QzRaL2PI0qSm/TBbyvQsPWPdfE8ZNlGduz4veya3A+7KfY8z91BDNsUxTnw5fDSb18/PSnP4X9b33rW9TX10dbt26lt7zlLZTL5eib3/wm3XPPPfS2t72NiIjuvvtuOu+882jLli10xRVXvOIKK4qiKIpyZvOKbD5yuV+/Lf7mv8mtW7dSo9GgdetsYq2VK1fS0NAQbd68+UXPUavVKJ/Pw5+iKIqiKK9fXvbLh+/7dPPNN9OVV15Jq1b9WiIYHR2lcDhMGeGZ0N/fT6Ojoy9yll/bkaTT6dbfwoULX/Q4RVEURVFeH7xsV9v169fTjh076NFHH33pg+fgtttuo1tvvbW1n8/n53wBmT7wFOw/8czB1nbUQc1z/gCeJ8w0rnhE6L5MW3ZFs0RF6OhUyup/kTim4352n9UySz5qlcWa1R9Lw6gNLr9gEez/nz+6tLWdSKF+XWUufq4IMe3IENgsjrwR+r4TYi59EdTovYYIx8zaJyq0dhO27TElXAyTKdRHvTDaBsxGOIx92SVCj6cytt17evugrM40UWk70i1sPiLMViEtQtPHWTjoqHA1E2YDtHy1tQ/Z8eQzUPa9Tf/b2h4M4zUWuFY7rcgxKZrq/KGh1nZPJ2qu3QO2DSoZLJueQE04G7CriwP9GAo+ytzRq1XsSyP+V0kk7b1096K9TCjC+k+4gEpizDYhJMJ3c+2/IWwzpCFDkNk8BMP4DPMw6X4Tn4O6wf0Kq690mR2etjZWo0W8fq5qv1eqYHj1Uhl1+gaz8zjOTIpsmUM4TxCamVCMpUUPixMZtu8IGw+Z5KDB5ommNNNiz5Aj2iooQrHPBXdOyMTxeQrGcI5rOLa+xsyeksGRJifiwXTYeMpWcRxue97amQTKIixDFcfzWy5Z0NruFbZQh0v2+Tq87yiUVQu2fWIJrNvqJfjshdzZ3cH9hh2HMZGC4PAkpgc4mLO/c9kGHjuVt+Op6eAc39klUnOw5//Azp1QtuSNl9Ir5WW9fGzYsIF+9KMf0SOPPEILFthOGRgYoHq9TtlsFlY/xsbGaGBg4EXPFYlEKBKJvGiZoiiKoiivP05KdjHG0IYNG+j++++nBx98kJYsWQLla9asoVAoRJs2bWp9tnv3bjp8+DCtXbv21NRYURRFUZQzmpNa+Vi/fj3dc8899P3vf586OjpadhzpdJpisRil02n62Mc+Rrfeeit1dXVRKpWiT37yk7R27dpT5umSG0OD1MqkXXcLC++fZBCX69JpviwropiyJaagWKKMxbCZ3IBdvps3D5fgmLJCDR+ljG4W4W5kBJcAl65aDPsdQ1auaNaFextbbg4K2ScYki6OdjmzUccl3FrBRtQrlnDZMUCzL18eK+B5poq2LT0R+e6CC0SnhE8sIqI8SrocJtJWPuntRSmlWrJjpFBAV+SqiIjosWEQEq6K/LvROMpH6U68Zom5Tq5+4xugrHj4WGv7yJ69UNbDsuzmy+ju54jIrQtWLGtt9y/AZeE6uy9f+B86YnnVMNfFThENNRy21/SmMepuUEhhPUxq6ZRu3Nwls4HLwhLu2R4Qrpx8xb1aRdfAmogMyiNfHrcaz07riee7IeSKGpNLSw2cJ3I1u1/0cExOs8y1hRzKNY0qjjuIqCwry/ZlmXwuAuyA4/+TZNKFaBEZfZnvNYV7PA81ID10T/BxJiKii976ltZ2RMihNeGBf3h0pLWdncE5JcDqFwngFxMdON+kO1mkXZFROsjCCewdwXniie3Y7+fMt5LEovno2j+Preo/+SxGbn3+sHWJX9KXgTK3gm3gMxdZR8iP/V0svMMRjJuVncH7yrIxmndQu51k8l/3AM4h8xeiq22DRT+eEtmvcdnh5XFSLx933XUXERFdddVV8Pndd99NH/7wh4mI6Mtf/jK5rks33HAD1Wo1uvrqq+nrX//6KaiqoiiKoiivB07q5cNIy6gXIRqN0p133kl33nnny66UoiiKoiivXzS3i6IoiqIobeWMy2o7sPJC2L98en9r2xWa9IIlqGnFmQdOICT0tqbVwrwGassuofbusgyeVUKt8LyLrHtvTmRwLeas7ts/iO6h5w4KuxKWcZaiaG8QdjKtbSHnky+1XJbp1w1iWSRq9UAng25WMoR6k7knVkaFhs9cnFeei2pgqk9k0j1AJ0StgXp+VWjmmYy1uRjsw7p7zMZglGnHREST4xiWvMmuUyjiffmu1VY94e7X1YP9l2I2IU3R76uusvZOD1eyULbtgLUB8cTK4qVrroR9N2Pti2oig3OjZm0MymV082w0pEuq7ZO0CE3vM3ds4Sl+nLuzO8exJ0OTWzL4+FxWPTvAsyW8j2lpV9HPvivOYxyrX/Pst0RErot2Ug1m51GrSXsZ+8x4Ht50iYViz4tsog3pbszGkwxRHmQZVANC+3eljQVLV+AbvGeP3WdDuNk3xXjmWb7lHMItQmSqCT9wYukSiIiWrFrd2p4YwfDqPcIbcmi+dSsvCXfnWNDe1xUXYyqBNVe8Cfb7Bqw35vAezEPmM4OvF555GspSKZEtN2fn4xFhedO53NahkUQbvMeOvtDa7hV2Wp6we2FJkckV4R0SzO3/UAXrVhQuxFXHusuPlETGZGYTeP6q86Bs2RIM9xBO2N+54VG06ToV6MqHoiiKoihtRV8+FEVRFEVpK/ryoSiKoihKWznjbD46578D9hNvfKi17QpBNBBOw74bs3qtHxSacMBq9q6HmrkjNEcnafW3QB2bcNkqawtQraB2uvtZe94ly1HjTIfRdqR0xIazdeMZKPNcrgeinUIwhJqjF7Up5QMiJkggzcKHp7GtjPD158ruOQMi5DOz+Qj46HMu7UxIpnCfhaTQThMxtE0wLGZJTeSfznTae+kVaat9Yc9TKdnvyrTnE+PWXqReL0FZs4nHdqRsv8djaHtU78q0ti++6regrH+51b7jcfTJv+BCtG/isVeODWOKay4fyzgfARGCP8NicogwFpTLZm29hZYcFaHxgyzMswxj32zOHVKdU29aDb8uxodxWMyCMt7XVE7E/Kn32+uLGDuGBRR3xD1L2w2etr7eFDZUjn3e8yWMy5JjcSRKou08Dy/qMDuKoEhX4LO2DMo4/iIwusOMbaTNUJPdV1O0q3wKfR5+XoYzd2bdIfe4yCOz8+hP/7u1PTM+DmUhUffeedZ2bt6i5VA2n5ke9afweb5yDdqApLutHUNfJ9qfxXw7H5+fwXm0o4S2YhE2r40cxvg3R6psrCVwXr/4He9qbQ8t74ay1Hw8tvCCfaYDIiz6yISta16UlUJR2D+ctXPjgQn8XekftCHd+0R8pHAEf8u60/Z3ppgVtnunAF35UBRFURSlrejLh6IoiqIobeWMk12Mg0tMQZaF0xHLzaaBS+V+mIVfDuEyvstCa8uw1r4Io2yYC1tUrEal2JL7MREGNxK1rktBg9JFo4pLeUeP2nsxzVEocwJ2iTKeQLfBVGYp7CcX2iV2X4ReN2zZ/LiMt3MspwbEMrXLpI1mDZf5HLGcGqiiLDMbqQ7s51hUuGCWs63tPEZfJuNZeSAaEeeJYPhlx7Ht5wvpoFqwdffreB8jo9gnPSyufpfInBuP2nZfMIhLreeeu8JeQ/RBNIryDXfX9ITrphOy0ko0ivJIKIjncZmrbb6EEqPPlvGPHwFiyZ25WboOljnOiS/HV5kkIVf8DQsfXhAyZraIz3eVtUldyBxBLskKSVF43lKduSYbMSaIybVl4Q5eZPs1eR++dJnl9RESMLtN7yVcmB1WeVc0Hr9EQGRClS673L3WF3XlodmlW7AbeIkK8vqUbX+tXoZSSiiB83EXkyS6RQbnnoi95sjzW6Hs6AvP4TXrdnKICPf0+qgN05AsHoSylAzBz0KfH8viefYftvEDFly2DsredYWdj/2pfVCWrYuUFmyu8so4KA8cteHNJ/GRpWNCBd83Yw8IpjAMwdB86+6bzWLI9KXLF8N+ccb+JlWnUCaLpzHUwMtBVz4URVEURWkr+vKhKIqiKEpb0ZcPRVEURVHayhln80EG7RaaFaZzGiHeOqipBR3rOuQKV1vyrV7s1PB7pi5Etoa1cQiUhZteweqaEWE7MtDLtFRh45HNoutmybffrYk08Ia5AvemUQ9tNFHHC6VsOPGgiy6XTthqjDJIshEhuX3DwlN7aG/gMzfUgHCxlLYJfhXvczbKRdQYR0fwe7WKbT+/H1NBB8nqkYGksPEQQ75csy528QjaSngT9prTk2hYklmMmqfDbBMq5aIos/0ejeL1oxE7nqV67gkbkCAbsz29GO44FLLndR28hgyvXmFuyg2hO/PQ4zKPZDCI5+Vp4aviPCdj81FjFwqIMcpNbariPmrCroOPtFoTy7htQr2O5ymLcPhNZvcSjuH4qTM7k6qIKe+xe/Zk7nkJm348d3b3cxnq3JGh2FlzBeWxzLDDSCMP0T8eze5qC8NQjElPutLPwRveuKq1HXFwHl9wDtqAmA47/0UIn/1g0fZXZwZtqMZnMKX9xMFnWtsDCXTfz45ZWw0j7IdmHOz3Izk7unaN4vjZN2rn3MQKdL++9LesLcu4GL/jo0dhv1a2c8zYEbSdmxjPtranhZHSc6NoR1dn9ozvv+F6KOuL2YG354U9WJ/xCdhPMDuhed043+Bdvjx05UNRFEVRlLaiLx+KoiiKorSVM092IXQb9LibkYPLp5EYvlvxpXEZ6ZI8u2zsVKXsgktghkkHvoi62NNpl/ZkdM+gYctjNRGNUHigdrAMveEwSheVql2SDIZxOSwQwi71mvY+XSEHBGqsDcLCZ9iTWTiZS58jjuWui9I3UPpOyuyes9DbhRKIMdgnE6N2ybRZw/tyWV3jscVQVhcSViGfbW2XKtiXex+xbnt1kb3znW/E6KPzFtjomo2GlDJsG3geLpl6TM6SskZQ9GWY9VE4jMvWdS57GKxrsyndBu1DUxfuojwyaVOUxWLYJ/x5Coj2CQRY3QsYiVTSYPKJ7wiXUBb9syGGVrkmotWy+vqEda3XmWwoJBkZ4dRhqaKlRFMs22e/IiUrJrU4IeGSepzqwcqFvmVAPpHPk5Q5WORWUeKyZXNHuNoa6RrNnlMZKZWnLG7ITL4kpO456GXZpzNJzNQdCOK8cGDfQXvNMvbz0gErn9d9zFId68BIzR0sA3g9i+7xjaZ9nqZqmEX2hRGckHeN2d+AA9M435SL9tiVwzuhLJK3rq3LetDtf7qO+zUm70+WUQYaY9md905h2WQZx+EVv22zaP/W2jdAWWXaytkvHNoPZU888RTsR9mzaCpoevCG91xHrxRd+VAURVEUpa3oy4eiKIqiKG1FXz4URVEURWkrZ57Nh4M6tBPiGSBRdw66qItzLd6ITKgBrjtLzVNo6G7A2nUE4sKdNmP3O6uoh04e2dvajomsrY7QKoMstHbTx7o2mlZzjKXwPPEutAEJs9DeXl5o7yzUuSu+ZwjbOcDbxEW9tslcyJwa3rMRdjhe/cTCqy+cj/XJ59HupjNjNf1mHeuan7Fh7UcDOMSLIrx7tWLbsnoEtdTisHXnDaTwntNJDFvM291vyqy/zOVR2HzUKva+HJFrVLqdGtYHRobSZkYFDTG2m6I+3O6kIUOEM5dDaV0g7TqazG5KutZ60mZoDrgHoiOe7yazVSg38fozRdS6vaatQ9jBYxseSwEg+sCVGWfZPJLNCVujads+jYZ0e7XXd0TqXNcRdiWsdY1w2eV9e9xcJMYIt2/ypB0HCw1vnLltPpr8mqLnPbZfF/WRoennIsAM9GpVdNaceAHdThsl+5wuO381lMXjtg5H8vjM1gpi3kpmWtv5Es4FlZC1x9hbxVQYz05mYX+yyG5UpCtYscI++91pbNfyuLVNK4hHYiaHc2PvQhuKfc/YMJTtPGrbZ1qk+3jzW9bC/rqrLm1td4TxGqGMdcM97/xVUDb96OOwf4y53noi/fUb6JWjKx+KoiiKorQVfflQFEVRFKWt6MuHoiiKoiht5Yyz+XCE7hyKW1uJQCMiDhYaKNO3XWFHwZVMT8QPMEIz9wM29K4TwVTQgZi1+Qi4qGvyLPGJhEh7HkXbDZfpijLVuuszO4GwsEeJoA2Kz+xcGjkMn1uYsqGI45V5UBaWgQl42nMX68r9040IcR8SsSsqpROz+ajWsQ86hM1FMmn98psi9kA8yexnZMyLAmr4CWa74UTxXbxZtnUNJERa+qB8dNh3hZ2JwyIwOMKOIx5ibSnHpOh3Pkh9T9g+sTHq+9LGQ4xfNiZykxgnIc/SaEdELJHjY0zYsWZElAlpSzIXGNdC2AyxazZFX/rCriMUsPV1RdvxWCLS5kOO0SA7T0mEXs8XWTsbHC9hFuv8uDA5wlYDEKHYuQmIf5w9iIwJYvHEI8sPNZ78noj9AjZveJ4mK2s08Z49/yTC6JftPFEqYR9EIxjzYuXCodb24LIFUHbwkLWHcJwuKFt6PoZp7+2088ZUD4Zip5j9rpk/AkXdy9EGpZqzz0lHBJ+L3k47N6Qi4tljcam2bN8BZfuPYiqMReyed0/iPDlat31w1bq3QdnvvOOtsJ+K2z4pFjEmyfiMtZEZncpCWU10fIOlafBk/o1TgK58KIqiKIrSVk7q5eOuu+6i1atXUyqVolQqRWvXrqWf/OQnrfJqtUrr16+n7u5uSiaTdMMNN9DY2NgcZ1QURVEU5WzjpGSXBQsW0B133EErVqwgYwx9+9vfpuuuu462bdtGF1xwAd1yyy304x//mO677z5Kp9O0YcMGuv766+lXv/rVKauwkW5xPKq0i7cjw0NTUy4b8+/adVJXuqUFcUmQu6kFhWugYa6cporuoamEXaZ2QyJjaUws1bMlXVfeB49mLiWiKoYQrueyre3JCVxKHC1YGSZWwXXihHBTdlgYcM9B+abh2To0xftsNIxhi6s1dt9zvPo2hWvX4CBmka2yEPeOyFDsRlj21y50ia0JN9TprG3bpljGL5Lty/7BJVB29NgR2M8x9+fuLlwKjiesTOcKOQtdVEWDyMywfNeX6/rMHVKMiexMFvbzOVvXehmXd6Mh2++xGI57mR2XWHhxTzyXxpy47ILuo7PLE8e5r8rmYvJFQ7oXN23dfSP6IChC1bO2zIsw11WWHkBKKwHm9i+7UsolUDZH5lpHuOia41QOlo1WSCAg3xwnpeB+g82NnrgIl1Y8Ibv4/on//2pYWoaaqEAwg8+M22nTFQyPoDxxZNjOY4uX4nO5cOki2J+eYPNhAPu5woZI/+BCKOuKiGcowLJoixQR9ZJ9Dp7ahXOsG7ay6mQRM+X+7869sP/ws1baCcVw3rz2d97d2v6D3/tdKKuJZy1Xss/36HAWyrY/a6/5/L4DUJZMYP3CKbufy+Jv2angpF4+rr32Wti//fbb6a677qItW7bQggUL6Jvf/Cbdc8899La3/VqTuvvuu+m8886jLVu20BVXXPFip1QURVEU5SzjZdt8eJ5H9957L5VKJVq7di1t3bqVGo0GrVu3rnXMypUraWhoiDZv3jzreWq1GuXzefhTFEVRFOX1y0m/fDz77LOUTCYpEonQJz7xCbr//vvp/PPPp9HRUQqHw5TJZOD4/v5+Gh0dffGTEdHGjRspnU63/hYuXDjrsYqiKIqinPmctKvtueeeS9u3b6dcLkf/+Z//STfddBM9/PDDL7sCt912G916662t/Xw+P/cLiDeN+5WZ1mZNhM7m6biJiMLMncuRLmI+C70uQyGHhJsl02Rd4TrpN1nK7RxqleRZVzPfE+99AbwGl7e9Errsekzjawjh2RNpvnPsu8dy6HY1WbB1Dx9n84FtwOVsJ4R6qBNg7RFCO4GqCIldqbMT4aHA0BCOAU9o+OkOq4m6wt04l8+2tiebeF+hMF6U2+yERKj8FSxM8cD8QSgrTOE4LBRsO5emZqCsr8/aq0g7ighz4T1O+xc2H9z1tlrGvizm7fUnJrBupSKmwyay7RWLo6t4kNl8eN7csbO5G2goJFMZnLjNB7drkHYdvA2kjUcshs9MlKUkIIP9zt2dSdj21EWM8DIbM0Wh79fqzHZEuBeHmK3EcRYe0nyH2f6ISPnQrg69lM2HbRRpy8KnBmnjcZzNB/OlFI8MnPe4cOpz2LJIfGZHJm2fpsQzU5myc2c8gv2czdqxvmwZ2niUqjg3jUzY83I7MSKiYMzaNMifg10HD2F9stnWdiSEthHxiA2bUHTRxqxat2XxfrRrWXEx3teeXXta21e++U1Q9v7ff19ruzONYQcefw5deMdm2FwwjA4fTtg+78vPOw/K8sI2jAL297RL/AaeCk765SMcDtPy5b/2pV6zZg098cQT9NWvfpXe//73U71ep2w2C6sfY2NjNDAwMMvZiCKRCEUip/7GFEVRFEV5bfKK43z4vk+1Wo3WrFlDoVCINm3a1CrbvXs3HT58mNauXTvHGRRFURRFOZs4qZWP2267ja655hoaGhqiQqFA99xzD/3yl7+kn/3sZ5ROp+ljH/sY3XrrrdTV1UWpVIo++clP0tq1a9XTRVEURVGUFif18jE+Pk4f+tCH6NixY5ROp2n16tX0s5/9jN7xjncQEdGXv/xlcl2XbrjhBqrVanT11VfT17/+9VNa4fwMxld4Yfszre1SHrXtcAy1uRALPR4JY3hzl4WS9Q3ajkhNNhJhIbED0ubDfrc6cwzKKh47r9BnQzEMc22MrY9poFbJ45fIVNmeSC9fYjE4siW8kUKJ6cUO3nNChNZOhKxWmEiiTGa43YQIUU7CbmAyy2xSZlfjKChsTrLTqAl3ZazuGQzhNSvMBmVyXAS5c9KwW6+xfgjjeRa8cWVrW1SHoiIkdzTObDmEvcPIwYOtbWkbEU3ZdoVz0PF2Aw1mz1Or4lgvFW08ilJJ2HgYucBp+yQgxm+NxQiRdhtNYSjAvyvtU+LClmQuAsw2KxjE+gRZDJegiOcSCmMfBIK8DjJ2hv2uDJ9SF+Hoi7wNhE1D3efxMPA8/IOgtM1wpT0Przgeyo9sCmMRXzxPTRZnQ9px1Bs87glevylszprN2c/D44cYaUsjJ8c5mJyxnozNCsZPKRTRPq47ap+Fpog/4bJx54r09gER66mzx8YLmRAxSSYn7DWnJ7NYn4Y4z/zzW9uD/ThxHTxg7UMqIn7TdN6OpSPD+HuwdNky2O/un9/afvNv4T/sETY3PPfcTijbuWMX7I8y+5TChPAeZTFtzrvgAijyRGylOrNvys+cei/Uk3r5+OY3vzlneTQapTvvvJPuvPPOV1QpRVEURVFev2huF0VRFEVR2soZl9X2hw/8N+w/9fBzrW0ZXtgTaR55+OWocNMLhqwM44p1WV+ElQ6yZT+RwJSaDR5yGpdzG8xPrVpDKcURrmcBxy6PBYQ7L2RxFW6Dngi167FlY9fgkn+V+dT5wsUxHkaX3VjALh/K5W6fZbKtywzAIux2o2Hvcw4nKGrUUDrwPazP0SM2s2VUSERN1s6ZNLq35fL4vl1lklY4LDICs/OEhUdWVYQwr+Z4Zl+RHZLJF7EYyn2Fig1b7IawXcNzuK82hT8kr3tYuMXJsOhcPmk08D7K5dmzDktphYcsn+ueX4owk80CQq9wA0x2EfJERB7LBAuZCZqPw4Zwn61UcWwVK3ZMlGtY1mQyg0zDwN1iA+LfOtl2XE2RLtaGyVAB6T4r9vlcIDIHUIPJLlyCISLyZJZb3lxCpgs4XLLCseRKH+I5mGIu4JWZcShbunQ+7PezLNZZkZk1lrSyd1hkFe/uxjQMkYh1yZ84hi7oo0wGGR1Bebbh47OY6LDP1MQMhhrPV23DHxnF1BPxhL0PmXV4pwhvPtjT29ref3gYygpFe96ZUeFKX8IxOnLMtm13El1/S2xsb3v6GSjrlHNlwc5psdiJy6gniq58KIqiKIrSVvTlQ1EURVGUtqIvH4qiKIqitBXHzJXr+TSQz+cpnU7TZz/7WY18qiiKoihnCLVaje644w7K5XKUSqXmPFZXPhRFURRFaSv68qEoiqIoSlvRlw9FURRFUdqKvnwoiqIoitJW9OVDURRFUZS28pqLcPob55uaiCCpKIqiKMprl9/8bp+IE+1rztX2yJEjtHDhwtNdDUVRFEVRXgbDw8O0YMGCOY95zb18+L5PIyMjZIyhoaEhGh4efkl/4bORfD5PCxcu1PaZBW2fudH2mRttn7nR9pmds7ltjDFUKBRocHCQXHduq47XnOziui4tWLCA8vk8ERGlUqmzrgNPBm2fudH2mRttn7nR9pkbbZ/ZOVvbJp1On9BxanCqKIqiKEpb0ZcPRVEURVHaymv25SMSidBf/uVfan6XWdD2mRttn7nR9pkbbZ+50faZHW2bE+M1Z3CqKIqiKMrrm9fsyoeiKIqiKK9P9OVDURRFUZS2oi8fiqIoiqK0FX35UBRFURSlrejLh6IoiqIobeU1+/Jx55130uLFiykajdLll19Ojz/++OmuUtvZuHEjXXrppdTR0UF9fX30vve9j3bv3g3HVKtVWr9+PXV3d1MymaQbbriBxsbGTlONTy933HEHOY5DN998c+uzs719jh49Sn/4h39I3d3dFIvF6MILL6Qnn3yyVW6MoS9+8Ys0b948isVitG7dOtq7d+9prHH78DyPvvCFL9CSJUsoFovRsmXL6K//+q8hKdbZ1D6PPPIIXXvttTQ4OEiO49ADDzwA5SfSFtPT03TjjTdSKpWiTCZDH/vYx6hYLLbxLl495mqfRqNBn/nMZ+jCCy+kRCJBg4OD9KEPfYhGRkbgHK/n9jlpzGuQe++914TDYfMv//Iv5rnnnjN//Md/bDKZjBkbGzvdVWsrV199tbn77rvNjh07zPbt28273/1uMzQ0ZIrFYuuYT3ziE2bhwoVm06ZN5sknnzRXXHGFedOb3nQaa316ePzxx83ixYvN6tWrzac+9anW52dz+0xPT5tFixaZD3/4w+axxx4z+/fvNz/72c/Mvn37WsfccccdJp1OmwceeMA8/fTT5r3vfa9ZsmSJqVQqp7Hm7eH222833d3d5kc/+pE5cOCAue+++0wymTRf/epXW8ecTe3zX//1X+bzn/+8+d73vmeIyNx///1QfiJt8a53vctcdNFFZsuWLeZ//ud/zPLly80HP/jBNt/Jq8Nc7ZPNZs26devMd7/7XbNr1y6zefNmc9lll5k1a9bAOV7P7XOyvCZfPi677DKzfv361r7neWZwcNBs3LjxNNbq9DM+Pm6IyDz88MPGmF8P+FAoZO67777WMc8//7whIrN58+bTVc22UygUzIoVK8zPf/5z89a3vrX18nG2t89nPvMZ8+Y3v3nWct/3zcDAgPn7v//71mfZbNZEIhHz7//+7+2o4mnlPe95j/noRz8Kn11//fXmxhtvNMac3e0jf1xPpC127txpiMg88cQTrWN+8pOfGMdxzNGjR9tW93bwYi9nkscff9wQkTl06JAx5uxqnxPhNSe71Ot12rp1K61bt671meu6tG7dOtq8efNprNnpJ5fLERFRV1cXERFt3bqVGo0GtNXKlStpaGjorGqr9evX03ve8x5oByJtnx/84Ad0ySWX0O///u9TX18fXXzxxfTP//zPrfIDBw7Q6OgotE86nabLL7/8rGifN73pTbRp0ybas2cPERE9/fTT9Oijj9I111xDRNo+nBNpi82bN1Mmk6FLLrmkdcy6devIdV167LHH2l7n000ulyPHcSiTyRCRto/kNZfVdnJykjzPo/7+fvi8v7+fdu3adZpqdfrxfZ9uvvlmuvLKK2nVqlVERDQ6OkrhcLg1uH9Df38/jY6OnoZatp97772XnnrqKXriiSeOKzvb22f//v1011130a233kqf+9zn6IknnqA/+7M/o3A4TDfddFOrDV7sWTsb2uezn/0s5fN5WrlyJQUCAfI8j26//Xa68cYbiYjO+vbhnEhbjI6OUl9fH5QHg0Hq6uo669qrWq3SZz7zGfrgBz/Yymyr7YO85l4+lBdn/fr1tGPHDnr00UdPd1VeMwwPD9OnPvUp+vnPf07RaPR0V+c1h+/7dMkll9Df/u3fEhHRxRdfTDt27KBvfOMbdNNNN53m2p1+/uM//oO+853v0D333EMXXHABbd++nW6++WYaHBzU9lFeNo1Gg/7gD/6AjDF01113ne7qvGZ5zckuPT09FAgEjvNIGBsbo4GBgdNUq9PLhg0b6Ec/+hE99NBDtGDBgtbnAwMDVK/XKZvNwvFnS1tt3bqVxsfH6Y1vfCMFg0EKBoP08MMP09e+9jUKBoPU399/VrfPvHnz6Pzzz4fPzjvvPDp8+DARUasNztZn7c///M/ps5/9LH3gAx+gCy+8kP7oj/6IbrnlFtq4cSMRaftwTqQtBgYGaHx8HMqbzSZNT0+fNe31mxePQ4cO0c9//vPWqgeRto/kNffyEQ6Hac2aNbRp06bWZ77v06ZNm2jt2rWnsWbtxxhDGzZsoPvvv58efPBBWrJkCZSvWbOGQqEQtNXu3bvp8OHDZ0Vbvf3tb6dnn32Wtm/f3vq75JJL6MYbb2xtn83tc+WVVx7nmr1nzx5atGgREREtWbKEBgYGoH3y+Tw99thjZ0X7lMtlcl2cAgOBAPm+T0TaPpwTaYu1a9dSNpulrVu3to558MEHyfd9uvzyy9te53bzmxePvXv30i9+8Qvq7u6G8rO9fY7jdFu8vhj33nuviUQi5lvf+pbZuXOn+fjHP24ymYwZHR093VVrK3/yJ39i0um0+eUvf2mOHTvW+iuXy61jPvGJT5ihoSHz4IMPmieffNKsXbvWrF279jTW+vTCvV2MObvb5/HHHzfBYNDcfvvtZu/eveY73/mOicfj5t/+7d9ax9xxxx0mk8mY73//++aZZ54x11133evWlVRy0003mfnz57dcbb/3ve+Znp4e8+lPf7p1zNnUPoVCwWzbts1s27bNEJH5h3/4B7Nt27aWt8aJtMW73vUuc/HFF5vHHnvMPProo2bFihWvG1fSudqnXq+b9773vWbBggVm+/btMF/XarXWOV7P7XOyvCZfPowx5h//8R/N0NCQCYfD5rLLLjNbtmw53VVqO0T0on93331365hKpWL+9E//1HR2dpp4PG5+93d/1xw7duz0Vfo0I18+zvb2+eEPf2hWrVplIpGIWblypfmnf/onKPd933zhC18w/f39JhKJmLe//e1m9+7dp6m27SWfz5tPfepTZmhoyESjUbN06VLz+c9/Hn4szqb2eeihh150vrnpppuMMSfWFlNTU+aDH/ygSSaTJpVKmY985COmUCichrs59czVPgcOHJh1vn7ooYda53g9t8/J4hjDwvkpiqIoiqK8yrzmbD4URVEURXl9oy8fiqIoiqK0FX35UBRFURSlrejLh6IoiqIobUVfPhRFURRFaSv68qEoiqIoSlvRlw9FURRFUdqKvnwoiqIoitJW9OVDURRFUZS2oi8fiqIoiqK0FX35UBRFURSlrfx/HMsEFumK76IAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "deer  dog   bird  frog \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# net1 = resnet34(weights=None)\n",
        "# net2 = wide_resnet50_2(weights=None)"
      ],
      "metadata": {
        "id": "0nXIdiHimBiT"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(\n",
        "            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,\n",
        "                               stride=1, padding=1, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "\n",
        "        self.shortcut = nn.Sequential()\n",
        "        if stride != 1 or in_planes != self.expansion*planes:\n",
        "            self.shortcut = nn.Sequential(\n",
        "                nn.Conv2d(in_planes, self.expansion*planes,\n",
        "                          kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm2d(self.expansion*planes)\n",
        "            )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.bn2(self.conv2(out))\n",
        "        out += self.shortcut(x)\n",
        "        out = F.relu(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "class Bottleneck(nn.Module):\n",
        "    expansion = 4\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(Bottleneck, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,\n",
        "                               stride=stride, padding=1, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "        self.conv3 = nn.Conv2d(planes, self.expansion *\n",
        "                               planes, kernel_size=1, bias=False)\n",
        "        self.bn3 = nn.BatchNorm2d(self.expansion*planes)\n",
        "\n",
        "        self.shortcut = nn.Sequential()\n",
        "        if stride != 1 or in_planes != self.expansion*planes:\n",
        "            self.shortcut = nn.Sequential(\n",
        "                nn.Conv2d(in_planes, self.expansion*planes,\n",
        "                          kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm2d(self.expansion*planes)\n",
        "            )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = F.relu(self.bn2(self.conv2(out)))\n",
        "        out = self.bn3(self.conv3(out))\n",
        "        out += self.shortcut(x)\n",
        "        out = F.relu(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, num_blocks, num_classes=10):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.in_planes = 64\n",
        "\n",
        "        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,\n",
        "                               stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(64)\n",
        "        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n",
        "        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n",
        "        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n",
        "        self.linear = nn.Linear(512*block.expansion, num_classes)\n",
        "\n",
        "    def _make_layer(self, block, planes, num_blocks, stride):\n",
        "        strides = [stride] + [1]*(num_blocks-1)\n",
        "        layers = []\n",
        "        for stride in strides:\n",
        "            layers.append(block(self.in_planes, planes, stride))\n",
        "            self.in_planes = planes * block.expansion\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.layer1(out)\n",
        "        out = self.layer2(out)\n",
        "        out = self.layer3(out)\n",
        "        out = self.layer4(out)\n",
        "        out = F.avg_pool2d(out, 4)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.linear(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "def ResNet18():\n",
        "    return ResNet(BasicBlock, [2, 2, 2, 2])\n",
        "\n",
        "\n",
        "def ResNet34():\n",
        "    return ResNet(BasicBlock, [3, 4, 6, 3])\n",
        "\n",
        "\n",
        "def ResNet50():\n",
        "    return ResNet(Bottleneck, [3, 4, 6, 3])\n",
        "\n",
        "\n",
        "def ResNet101():\n",
        "    return ResNet(Bottleneck, [3, 4, 23, 3])\n",
        "\n",
        "\n",
        "def ResNet152():\n",
        "    return ResNet(Bottleneck, [3, 8, 36, 3])\n",
        "\n"
      ],
      "metadata": {
        "id": "V8apdRSHCoqM"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Block(nn.Module):\n",
        "    '''Depthwise conv + Pointwise conv'''\n",
        "    def __init__(self, in_planes, out_planes, stride=1):\n",
        "        super(Block, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=in_planes, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(in_planes)\n",
        "        self.conv2 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(out_planes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = F.relu(self.bn2(self.conv2(out)))\n",
        "        return out\n",
        "\n",
        "\n",
        "class MobileNet(nn.Module):\n",
        "    # (128,2) means conv planes=128, conv stride=2, by default conv stride=1\n",
        "    cfg = [64, (128,2), 128, (256,2), 256, (512,2), 512, 512, 512, 512, 512, (1024,2), 1024]\n",
        "\n",
        "    def __init__(self, num_classes=10):\n",
        "        super(MobileNet, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(32)\n",
        "        self.layers = self._make_layers(in_planes=32)\n",
        "        self.linear = nn.Linear(1024, num_classes)\n",
        "\n",
        "    def _make_layers(self, in_planes):\n",
        "        layers = []\n",
        "        for x in self.cfg:\n",
        "            out_planes = x if isinstance(x, int) else x[0]\n",
        "            stride = 1 if isinstance(x, int) else x[1]\n",
        "            layers.append(Block(in_planes, out_planes, stride))\n",
        "            in_planes = out_planes\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.layers(out)\n",
        "        out = F.avg_pool2d(out, 2)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.linear(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "def test():\n",
        "    net = MobileNet()\n",
        "    x = torch.randn(1,3,32,32)\n",
        "    y = net(x)\n",
        "    print(y.size())"
      ],
      "metadata": {
        "id": "sK2lYN5sO0Nx"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "net1 = ResNet34()\n",
        "net2 = MobileNet()"
      ],
      "metadata": {
        "id": "z0OWSNuNCtjs"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer1 = optim.SGD(net1.parameters(), lr=0.001, momentum=0.9)\n",
        "optimizer2 = optim.SGD(net2.parameters(), lr=0.001, momentum=0.9)"
      ],
      "metadata": {
        "id": "6YaoGKmjmjAE"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_model(trainloader, net, criterion, optimizer, epochs, device):\n",
        "    net.to(device)\n",
        "    print_interval = int(len(trainloader)/5)\n",
        "\n",
        "    for epoch in range(epochs):  # loop over the dataset multiple times\n",
        "        running_loss = 0.0\n",
        "        for i, data in enumerate(trainloader, 0):\n",
        "            # get the inputs; data is a list of [inputs, labels]\n",
        "            inputs, labels = data\n",
        "            inputs, labels = inputs.to(device), labels.to(device)\n",
        "\n",
        "            # zero the parameter gradients\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # forward + backward + optimize\n",
        "            outputs = net(inputs)\n",
        "            loss = criterion(outputs, labels)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # print statistics\n",
        "            running_loss += loss.item()\n",
        "            if i % print_interval == print_interval-1:    # print every 2000 mini-batches\n",
        "                print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')\n",
        "                running_loss = 0.0\n",
        "\n",
        "    print('Finished Training')"
      ],
      "metadata": {
        "id": "_C_dXa6hqD7s"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 20\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)\n",
        "train_model(trainloader_cifar10, net1, criterion, optimizer1, epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oa1vm_iKrbrs",
        "outputId": "944d0aaa-f6a1-4cb1-acc4-84da1734a8a9"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n",
            "[1,  2500] loss: 2.491\n",
            "[1,  5000] loss: 1.969\n",
            "[1,  7500] loss: 1.706\n",
            "[1, 10000] loss: 1.493\n",
            "[1, 12500] loss: 1.344\n",
            "[2,  2500] loss: 1.186\n",
            "[2,  5000] loss: 1.145\n",
            "[2,  7500] loss: 1.052\n",
            "[2, 10000] loss: 1.009\n",
            "[2, 12500] loss: 0.954\n",
            "[3,  2500] loss: 0.840\n",
            "[3,  5000] loss: 0.790\n",
            "[3,  7500] loss: 0.808\n",
            "[3, 10000] loss: 0.775\n",
            "[3, 12500] loss: 0.764\n",
            "[4,  2500] loss: 0.625\n",
            "[4,  5000] loss: 0.620\n",
            "[4,  7500] loss: 0.619\n",
            "[4, 10000] loss: 0.602\n",
            "[4, 12500] loss: 0.627\n",
            "[5,  2500] loss: 0.464\n",
            "[5,  5000] loss: 0.497\n",
            "[5,  7500] loss: 0.491\n",
            "[5, 10000] loss: 0.483\n",
            "[5, 12500] loss: 0.511\n",
            "[6,  2500] loss: 0.334\n",
            "[6,  5000] loss: 0.365\n",
            "[6,  7500] loss: 0.395\n",
            "[6, 10000] loss: 0.388\n",
            "[6, 12500] loss: 0.392\n",
            "[7,  2500] loss: 0.256\n",
            "[7,  5000] loss: 0.265\n",
            "[7,  7500] loss: 0.275\n",
            "[7, 10000] loss: 0.308\n",
            "[7, 12500] loss: 0.308\n",
            "[8,  2500] loss: 0.176\n",
            "[8,  5000] loss: 0.216\n",
            "[8,  7500] loss: 0.213\n",
            "[8, 10000] loss: 0.231\n",
            "[8, 12500] loss: 0.241\n",
            "[9,  2500] loss: 0.125\n",
            "[9,  5000] loss: 0.141\n",
            "[9,  7500] loss: 0.164\n",
            "[9, 10000] loss: 0.171\n",
            "[9, 12500] loss: 0.176\n",
            "[10,  2500] loss: 0.102\n",
            "[10,  5000] loss: 0.113\n",
            "[10,  7500] loss: 0.116\n",
            "[10, 10000] loss: 0.121\n",
            "[10, 12500] loss: 0.154\n",
            "[11,  2500] loss: 0.068\n",
            "[11,  5000] loss: 0.083\n",
            "[11,  7500] loss: 0.076\n",
            "[11, 10000] loss: 0.094\n",
            "[11, 12500] loss: 0.117\n",
            "[12,  2500] loss: 0.056\n",
            "[12,  5000] loss: 0.045\n",
            "[12,  7500] loss: 0.057\n",
            "[12, 10000] loss: 0.067\n",
            "[12, 12500] loss: 0.089\n",
            "[13,  2500] loss: 0.060\n",
            "[13,  5000] loss: 0.063\n",
            "[13,  7500] loss: 0.055\n",
            "[13, 10000] loss: 0.067\n",
            "[13, 12500] loss: 0.061\n",
            "[14,  2500] loss: 0.030\n",
            "[14,  5000] loss: 0.040\n",
            "[14,  7500] loss: 0.027\n",
            "[14, 10000] loss: 0.049\n",
            "[14, 12500] loss: 0.044\n",
            "[15,  2500] loss: 0.031\n",
            "[15,  5000] loss: 0.048\n",
            "[15,  7500] loss: 0.049\n",
            "[15, 10000] loss: 0.037\n",
            "[15, 12500] loss: 0.050\n",
            "[16,  2500] loss: 0.029\n",
            "[16,  5000] loss: 0.031\n",
            "[16,  7500] loss: 0.025\n",
            "[16, 10000] loss: 0.028\n",
            "[16, 12500] loss: 0.024\n",
            "[17,  2500] loss: 0.016\n",
            "[17,  5000] loss: 0.017\n",
            "[17,  7500] loss: 0.017\n",
            "[17, 10000] loss: 0.015\n",
            "[17, 12500] loss: 0.027\n",
            "[18,  2500] loss: 0.018\n",
            "[18,  5000] loss: 0.022\n",
            "[18,  7500] loss: 0.016\n",
            "[18, 10000] loss: 0.023\n",
            "[18, 12500] loss: 0.021\n",
            "[19,  2500] loss: 0.017\n",
            "[19,  5000] loss: 0.015\n",
            "[19,  7500] loss: 0.026\n",
            "[19, 10000] loss: 0.023\n",
            "[19, 12500] loss: 0.019\n",
            "[20,  2500] loss: 0.019\n",
            "[20,  5000] loss: 0.017\n",
            "[20,  7500] loss: 0.017\n",
            "[20, 10000] loss: 0.013\n",
            "[20, 12500] loss: 0.009\n",
            "Finished Training\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(device)\n",
        "train_model(trainloader_cifar10, net2, criterion, optimizer2, epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FM3vp0_HzPAS",
        "outputId": "f94c9a1b-8c7f-4ab0-df10-1971580e4813"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n",
            "[1,  2500] loss: 2.740\n",
            "[1,  5000] loss: 2.462\n",
            "[1,  7500] loss: 2.328\n",
            "[1, 10000] loss: 2.191\n",
            "[1, 12500] loss: 2.119\n",
            "[2,  2500] loss: 2.009\n",
            "[2,  5000] loss: 1.925\n",
            "[2,  7500] loss: 1.829\n",
            "[2, 10000] loss: 1.765\n",
            "[2, 12500] loss: 1.702\n",
            "[3,  2500] loss: 1.590\n",
            "[3,  5000] loss: 1.536\n",
            "[3,  7500] loss: 1.528\n",
            "[3, 10000] loss: 1.425\n",
            "[3, 12500] loss: 1.426\n",
            "[4,  2500] loss: 1.296\n",
            "[4,  5000] loss: 1.333\n",
            "[4,  7500] loss: 1.292\n",
            "[4, 10000] loss: 1.281\n",
            "[4, 12500] loss: 1.218\n",
            "[5,  2500] loss: 1.109\n",
            "[5,  5000] loss: 1.124\n",
            "[5,  7500] loss: 1.134\n",
            "[5, 10000] loss: 1.114\n",
            "[5, 12500] loss: 1.121\n",
            "[6,  2500] loss: 0.965\n",
            "[6,  5000] loss: 0.983\n",
            "[6,  7500] loss: 0.996\n",
            "[6, 10000] loss: 1.020\n",
            "[6, 12500] loss: 1.002\n",
            "[7,  2500] loss: 0.856\n",
            "[7,  5000] loss: 0.880\n",
            "[7,  7500] loss: 0.897\n",
            "[7, 10000] loss: 0.889\n",
            "[7, 12500] loss: 0.912\n",
            "[8,  2500] loss: 0.740\n",
            "[8,  5000] loss: 0.809\n",
            "[8,  7500] loss: 0.807\n",
            "[8, 10000] loss: 0.808\n",
            "[8, 12500] loss: 0.825\n",
            "[9,  2500] loss: 0.641\n",
            "[9,  5000] loss: 0.720\n",
            "[9,  7500] loss: 0.734\n",
            "[9, 10000] loss: 0.737\n",
            "[9, 12500] loss: 0.766\n",
            "[10,  2500] loss: 0.582\n",
            "[10,  5000] loss: 0.646\n",
            "[10,  7500] loss: 0.664\n",
            "[10, 10000] loss: 0.709\n",
            "[10, 12500] loss: 0.687\n",
            "[11,  2500] loss: 0.538\n",
            "[11,  5000] loss: 0.568\n",
            "[11,  7500] loss: 0.619\n",
            "[11, 10000] loss: 0.660\n",
            "[11, 12500] loss: 0.629\n",
            "[12,  2500] loss: 0.482\n",
            "[12,  5000] loss: 0.538\n",
            "[12,  7500] loss: 0.555\n",
            "[12, 10000] loss: 0.599\n",
            "[12, 12500] loss: 0.595\n",
            "[13,  2500] loss: 0.433\n",
            "[13,  5000] loss: 0.485\n",
            "[13,  7500] loss: 0.525\n",
            "[13, 10000] loss: 0.510\n",
            "[13, 12500] loss: 0.536\n",
            "[14,  2500] loss: 0.397\n",
            "[14,  5000] loss: 0.430\n",
            "[14,  7500] loss: 0.458\n",
            "[14, 10000] loss: 0.506\n",
            "[14, 12500] loss: 0.512\n",
            "[15,  2500] loss: 0.354\n",
            "[15,  5000] loss: 0.415\n",
            "[15,  7500] loss: 0.420\n",
            "[15, 10000] loss: 0.446\n",
            "[15, 12500] loss: 0.480\n",
            "[16,  2500] loss: 0.311\n",
            "[16,  5000] loss: 0.363\n",
            "[16,  7500] loss: 0.391\n",
            "[16, 10000] loss: 0.405\n",
            "[16, 12500] loss: 0.433\n",
            "[17,  2500] loss: 0.310\n",
            "[17,  5000] loss: 0.326\n",
            "[17,  7500] loss: 0.341\n",
            "[17, 10000] loss: 0.386\n",
            "[17, 12500] loss: 0.401\n",
            "[18,  2500] loss: 0.260\n",
            "[18,  5000] loss: 0.299\n",
            "[18,  7500] loss: 0.330\n",
            "[18, 10000] loss: 0.342\n",
            "[18, 12500] loss: 0.361\n",
            "[19,  2500] loss: 0.246\n",
            "[19,  5000] loss: 0.270\n",
            "[19,  7500] loss: 0.296\n",
            "[19, 10000] loss: 0.321\n",
            "[19, 12500] loss: 0.339\n",
            "[20,  2500] loss: 0.213\n",
            "[20,  5000] loss: 0.224\n",
            "[20,  7500] loss: 0.277\n",
            "[20, 10000] loss: 0.292\n",
            "[20, 12500] loss: 0.311\n",
            "Finished Training\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def test_model(testloader, net, device):\n",
        "\n",
        "    net.to(device)\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    net.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for data in testloader:\n",
        "            images, labels = data\n",
        "            images, labels = images.to(device), labels.to(device)\n",
        "\n",
        "            outputs = net(images)\n",
        "            \n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "\n",
        "    acc = 100 * correct / total \n",
        "    return acc"
      ],
      "metadata": {
        "id": "V_ck4lRPsMjJ"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_model(testloader_cifar10, net1, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E_UmRYUbs78Q",
        "outputId": "2f32efb4-d72b-4084-f77f-25ac3730119c"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "85.63"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_model(testloader_cifar10, net2, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fExp-moBzhEv",
        "outputId": "0ed2e772-dd95-4851-9a39-7d8ad794fff0"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "75.7"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def mutual_learning(trainloader, net1, net2, criterion, optimizer1, optimizer2, epochs, device):\n",
        "    net1.to(device)\n",
        "    net2.to(device)\n",
        "    print_interval = int(len(trainloader)/5)\n",
        "    # kl_div = nn.KLDivLoss(reduction=\"sum\")\n",
        "\n",
        "    for epoch in range(epochs):  # loop over the dataset multiple times\n",
        "        running_loss1 = 0.0\n",
        "        running_loss2 = 0.0\n",
        "        for i, data in enumerate(trainloader, 0):\n",
        "            # get the inputs; data is a list of [inputs, labels]\n",
        "            inputs, labels = data\n",
        "            inputs, labels = inputs.to(device), labels.to(device)\n",
        "\n",
        "            # zero the parameter gradients\n",
        "            optimizer1.zero_grad()\n",
        "            optimizer2.zero_grad()\n",
        "\n",
        "            # forward + backward + optimize\n",
        "            outputs1 = net1(inputs)\n",
        "            # print(outputs1)\n",
        "            outputs2 = net2(inputs)\n",
        "            # print(outputs2.shape)\n",
        "            # print(labels.shape)\n",
        "\n",
        "            loss1 = criterion(outputs1, labels) + F.kl_div(F.softmax(outputs1, dim = 1), F.softmax(outputs2, dim = 1), reduction = 'sum') # add the kl term for mutual learning\n",
        "            loss1.backward(retain_graph=True)\n",
        "            optimizer1.step()\n",
        "\n",
        "            loss2 = criterion(outputs2, labels) + F.kl_div(F.softmax(outputs2, dim = 1), F.softmax(outputs1, dim = 1), reduction = 'sum') \n",
        "            loss2.backward()\n",
        "            optimizer2.step()\n",
        "\n",
        "            # print statistics\n",
        "            running_loss1 += loss1.item()\n",
        "            running_loss2 += loss2.item()\n",
        "            if i % print_interval == print_interval-1:    # print every 2000 mini-batches\n",
        "                print(f'[{epoch + 1}, {i + 1:5d}] loss1: {running_loss1 / 2000:.3f}')\n",
        "                print(f'[{epoch + 1}, {i + 1:5d}] loss2: {running_loss2 / 2000:.3f}')\n",
        "                running_loss1 = 0.0\n",
        "                running_loss2 = 0.0\n",
        "\n",
        "    print('Finished Training')"
      ],
      "metadata": {
        "id": "j7nJKwmFxhJ3"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 20\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "net1_mul = ResNet34()\n",
        "net2_mul = MobileNet()\n",
        "print(device)\n",
        "optimizer1_mul = optim.SGD(net1_mul.parameters(), lr=0.001, momentum=0.9)\n",
        "optimizer2_mul = optim.SGD(net2_mul.parameters(), lr=0.001, momentum=0.9)\n",
        "mutual_learning(trainloader_cifar10, net1_mul, net2_mul, criterion, optimizer1_mul, optimizer2_mul, epochs, device)\n",
        "\n",
        "# print('mutual net1')\n",
        "# test_model(testloader_cifar10, net1_mul, device)\n",
        "print('mutual net2')\n",
        "test_model(testloader_cifar10, net2_mul, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "APnXH_QnvVwp",
        "outputId": "539fc7db-29d5-4cb5-fb72-7e2c46a3d65d"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n",
            "[1,  2500] loss1: -9.244\n",
            "[1,  2500] loss2: -6.045\n",
            "[1,  5000] loss1: -9.721\n",
            "[1,  5000] loss2: -5.078\n",
            "[1,  7500] loss1: -9.957\n",
            "[1,  7500] loss2: -4.276\n",
            "[1, 10000] loss1: -10.115\n",
            "[1, 10000] loss2: -3.739\n",
            "[1, 12500] loss1: -10.237\n",
            "[1, 12500] loss2: -3.339\n",
            "[2,  2500] loss1: -10.375\n",
            "[2,  2500] loss2: -2.947\n",
            "[2,  5000] loss1: -10.419\n",
            "[2,  5000] loss2: -2.827\n",
            "[2,  7500] loss1: -10.489\n",
            "[2,  7500] loss2: -2.590\n",
            "[2, 10000] loss1: -10.523\n",
            "[2, 10000] loss2: -2.548\n",
            "[2, 12500] loss1: -10.539\n",
            "[2, 12500] loss2: -2.483\n",
            "[3,  2500] loss1: -10.640\n",
            "[3,  2500] loss2: -2.290\n",
            "[3,  5000] loss1: -10.637\n",
            "[3,  5000] loss2: -2.233\n",
            "[3,  7500] loss1: -10.668\n",
            "[3,  7500] loss2: -2.183\n",
            "[3, 10000] loss1: -10.684\n",
            "[3, 10000] loss2: -2.217\n",
            "[3, 12500] loss1: -10.684\n",
            "[3, 12500] loss2: -2.183\n",
            "[4,  2500] loss1: -10.795\n",
            "[4,  2500] loss2: -2.008\n",
            "[4,  5000] loss1: -10.770\n",
            "[4,  5000] loss2: -2.012\n",
            "[4,  7500] loss1: -10.785\n",
            "[4,  7500] loss2: -2.039\n",
            "[4, 10000] loss1: -10.776\n",
            "[4, 10000] loss2: -2.044\n",
            "[4, 12500] loss1: -10.778\n",
            "[4, 12500] loss2: -2.040\n",
            "[5,  2500] loss1: -10.887\n",
            "[5,  2500] loss2: -1.897\n",
            "[5,  5000] loss1: -10.892\n",
            "[5,  5000] loss2: -1.842\n",
            "[5,  7500] loss1: -10.875\n",
            "[5,  7500] loss2: -1.878\n",
            "[5, 10000] loss1: -10.868\n",
            "[5, 10000] loss2: -1.857\n",
            "[5, 12500] loss1: -10.855\n",
            "[5, 12500] loss2: -1.946\n",
            "[6,  2500] loss1: -10.984\n",
            "[6,  2500] loss2: -1.771\n",
            "[6,  5000] loss1: -10.967\n",
            "[6,  5000] loss2: -1.701\n",
            "[6,  7500] loss1: -10.941\n",
            "[6,  7500] loss2: -1.751\n",
            "[6, 10000] loss1: -10.947\n",
            "[6, 10000] loss2: -1.768\n",
            "[6, 12500] loss1: -10.926\n",
            "[6, 12500] loss2: -1.801\n",
            "[7,  2500] loss1: -11.052\n",
            "[7,  2500] loss2: -1.646\n",
            "[7,  5000] loss1: -11.031\n",
            "[7,  5000] loss2: -1.585\n",
            "[7,  7500] loss1: -11.013\n",
            "[7,  7500] loss2: -1.626\n",
            "[7, 10000] loss1: -11.009\n",
            "[7, 10000] loss2: -1.624\n",
            "[7, 12500] loss1: -11.014\n",
            "[7, 12500] loss2: -1.625\n",
            "[8,  2500] loss1: -11.077\n",
            "[8,  2500] loss2: -1.563\n",
            "[8,  5000] loss1: -11.083\n",
            "[8,  5000] loss2: -1.536\n",
            "[8,  7500] loss1: -11.061\n",
            "[8,  7500] loss2: -1.536\n",
            "[8, 10000] loss1: -11.073\n",
            "[8, 10000] loss2: -1.517\n",
            "[8, 12500] loss1: -11.067\n",
            "[8, 12500] loss2: -1.501\n",
            "[9,  2500] loss1: -11.118\n",
            "[9,  2500] loss2: -1.489\n",
            "[9,  5000] loss1: -11.090\n",
            "[9,  5000] loss2: -1.494\n",
            "[9,  7500] loss1: -11.111\n",
            "[9,  7500] loss2: -1.454\n",
            "[9, 10000] loss1: -11.089\n",
            "[9, 10000] loss2: -1.485\n",
            "[9, 12500] loss1: -11.099\n",
            "[9, 12500] loss2: -1.480\n",
            "[10,  2500] loss1: -11.148\n",
            "[10,  2500] loss2: -1.472\n",
            "[10,  5000] loss1: -11.134\n",
            "[10,  5000] loss2: -1.426\n",
            "[10,  7500] loss1: -11.120\n",
            "[10,  7500] loss2: -1.434\n",
            "[10, 10000] loss1: -11.129\n",
            "[10, 10000] loss2: -1.411\n",
            "[10, 12500] loss1: -11.129\n",
            "[10, 12500] loss2: -1.405\n",
            "[11,  2500] loss1: -11.153\n",
            "[11,  2500] loss2: -1.424\n",
            "[11,  5000] loss1: -11.130\n",
            "[11,  5000] loss2: -1.423\n",
            "[11,  7500] loss1: -11.147\n",
            "[11,  7500] loss2: -1.397\n",
            "[11, 10000] loss1: -11.153\n",
            "[11, 10000] loss2: -1.363\n",
            "[11, 12500] loss1: -11.130\n",
            "[11, 12500] loss2: -1.427\n",
            "[12,  2500] loss1: -11.159\n",
            "[12,  2500] loss2: -1.468\n",
            "[12,  5000] loss1: -11.166\n",
            "[12,  5000] loss2: -1.376\n",
            "[12,  7500] loss1: -11.150\n",
            "[12,  7500] loss2: -1.412\n",
            "[12, 10000] loss1: -11.157\n",
            "[12, 10000] loss2: -1.370\n",
            "[12, 12500] loss1: -11.145\n",
            "[12, 12500] loss2: -1.366\n",
            "[13,  2500] loss1: -11.152\n",
            "[13,  2500] loss2: -1.476\n",
            "[13,  5000] loss1: -11.166\n",
            "[13,  5000] loss2: -1.409\n",
            "[13,  7500] loss1: -11.156\n",
            "[13,  7500] loss2: -1.396\n",
            "[13, 10000] loss1: -11.154\n",
            "[13, 10000] loss2: -1.386\n",
            "[13, 12500] loss1: -11.162\n",
            "[13, 12500] loss2: -1.359\n",
            "[14,  2500] loss1: -11.154\n",
            "[14,  2500] loss2: -1.482\n",
            "[14,  5000] loss1: -11.155\n",
            "[14,  5000] loss2: -1.451\n",
            "[14,  7500] loss1: -11.167\n",
            "[14,  7500] loss2: -1.407\n",
            "[14, 10000] loss1: -11.159\n",
            "[14, 10000] loss2: -1.380\n",
            "[14, 12500] loss1: -11.165\n",
            "[14, 12500] loss2: -1.372\n",
            "[15,  2500] loss1: -11.156\n",
            "[15,  2500] loss2: -1.511\n",
            "[15,  5000] loss1: -11.151\n",
            "[15,  5000] loss2: -1.466\n",
            "[15,  7500] loss1: -11.167\n",
            "[15,  7500] loss2: -1.441\n",
            "[15, 10000] loss1: -11.170\n",
            "[15, 10000] loss2: -1.360\n",
            "[15, 12500] loss1: -11.172\n",
            "[15, 12500] loss2: -1.387\n",
            "[16,  2500] loss1: -11.160\n",
            "[16,  2500] loss2: -1.522\n",
            "[16,  5000] loss1: -11.153\n",
            "[16,  5000] loss2: -1.477\n",
            "[16,  7500] loss1: -11.160\n",
            "[16,  7500] loss2: -1.450\n",
            "[16, 10000] loss1: -11.164\n",
            "[16, 10000] loss2: -1.413\n",
            "[16, 12500] loss1: -11.154\n",
            "[16, 12500] loss2: -1.446\n",
            "[17,  2500] loss1: -11.153\n",
            "[17,  2500] loss2: -1.550\n",
            "[17,  5000] loss1: -11.153\n",
            "[17,  5000] loss2: -1.508\n",
            "[17,  7500] loss1: -11.158\n",
            "[17,  7500] loss2: -1.461\n",
            "[17, 10000] loss1: -11.158\n",
            "[17, 10000] loss2: -1.463\n",
            "[17, 12500] loss1: -11.160\n",
            "[17, 12500] loss2: -1.444\n",
            "[18,  2500] loss1: -11.145\n",
            "[18,  2500] loss2: -1.590\n",
            "[18,  5000] loss1: -11.144\n",
            "[18,  5000] loss2: -1.526\n",
            "[18,  7500] loss1: -11.143\n",
            "[18,  7500] loss2: -1.541\n",
            "[18, 10000] loss1: -11.146\n",
            "[18, 10000] loss2: -1.481\n",
            "[18, 12500] loss1: -11.171\n",
            "[18, 12500] loss2: -1.437\n",
            "[19,  2500] loss1: -11.138\n",
            "[19,  2500] loss2: -1.618\n",
            "[19,  5000] loss1: -11.139\n",
            "[19,  5000] loss2: -1.553\n",
            "[19,  7500] loss1: -11.148\n",
            "[19,  7500] loss2: -1.566\n",
            "[19, 10000] loss1: -11.140\n",
            "[19, 10000] loss2: -1.530\n",
            "[19, 12500] loss1: -11.147\n",
            "[19, 12500] loss2: -1.493\n",
            "[20,  2500] loss1: -11.126\n",
            "[20,  2500] loss2: -1.645\n",
            "[20,  5000] loss1: -11.138\n",
            "[20,  5000] loss2: -1.587\n",
            "[20,  7500] loss1: -11.143\n",
            "[20,  7500] loss2: -1.576\n",
            "[20, 10000] loss1: -11.138\n",
            "[20, 10000] loss2: -1.551\n",
            "[20, 12500] loss1: -11.154\n",
            "[20, 12500] loss2: -1.520\n",
            "Finished Training\n",
            "mutual net1\n",
            "mutual net2\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "78.46"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_model(testloader_cifar10, net1_mul, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r5-NfqPU1udE",
        "outputId": "b17952ca-4782-40c4-f459-1af96765431e"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "85.63"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_model(testloader_cifar10, net2_mul, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rlv72D_u2AIa",
        "outputId": "ff4d6168-888b-46e6-840f-b701132aaa05"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "78.46"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "V6EYBkuJ4rJt"
      },
      "execution_count": 18,
      "outputs": []
    }
  ]
}