DenseNet_Feature_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DenseNet Feature extraction.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shaam93/Building-a-classifer-with-Pytorch/blob/master/DenseNet_Feature_extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "whtAkstX9aqg",
        "colab_type": "code",
        "outputId": "bfa5021a-318b-42d2-8f7d-a41b8f5e7487",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        }
      },
      "source": [
        "# !pip install torch==0.4.0 torchvision\n",
        "!pip install torch torchvision\n",
        "import torch\n",
        "# this should print \"torch version: 0.4.0\"\n",
        "print(\"torch version: \", torch.__version__)\n",
        "\n",
        "# we need pillow version 5.3.0\n",
        "# we will uninstall the older version first\n",
        "!pip uninstall -y Pillow\n",
        "# then install the new one\n",
        "!pip install Pillow==5.3.0\n",
        "# import the new PIL version\n",
        "import PIL\n",
        "# this should print \"pillow version: 5.3.0\" \n",
        "# If it doesn't restart your runtime by going to Menu, Runtime, Restart Runtime\n",
        "print(\"pillow version: \", PIL.PILLOW_VERSION)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.1.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.3.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.16.4)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (5.3.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.12.0)\n",
            "torch version:  1.1.0\n",
            "Uninstalling Pillow-5.3.0:\n",
            "  Successfully uninstalled Pillow-5.3.0\n",
            "Collecting Pillow==5.3.0\n",
            "  Using cached https://files.pythonhosted.org/packages/62/94/5430ebaa83f91cc7a9f687ff5238e26164a779cca2ef9903232268b0a318/Pillow-5.3.0-cp36-cp36m-manylinux1_x86_64.whl\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: Pillow\n",
            "Successfully installed Pillow-5.3.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "pillow version:  5.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "twUy3qER-Un-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import json\n",
        "import copy\n",
        "import torch\n",
        "import torchvision\n",
        "from PIL import Image\n",
        "import numpy as np\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch import nn, optim\n",
        "from torch.optim import lr_scheduler\n",
        "from torchvision import transforms, models, datasets"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "deKRNzw5-eXS",
        "colab_type": "code",
        "outputId": "59bbd8bf-fdc7-4961-f493-5fb401b1e39b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 189
        }
      },
      "source": [
        "#to mount your Google Drive, other option is to zip the folder of images you need then upload it to Google colab with the \"Upload\" button, and then unzipping the file here with code\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PJP5-WDV_DxJ",
        "colab_type": "code",
        "outputId": "73566a70-2a4e-453a-be1a-bdd9c716a71d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# TEST THE MODEL WITH VGG16 (PRETRAINED=FALSE)\n",
        "\n",
        "# Test For CUDA\n",
        "\n",
        "# check if CUDA is available\n",
        "train_on_gpu = torch.cuda.is_available()\n",
        "\n",
        "if not train_on_gpu:\n",
        "    print('CUDA is not available.  Training on CPU ...')\n",
        "else:\n",
        "    print('CUDA is available!  Training on GPU ...')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CUDA is available!  Training on GPU ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nxpfuwc-_gZO",
        "colab_type": "code",
        "outputId": "6f9e62c4-a966-4054-8c07-ed5ec562e9f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Load the Data\n",
        "from torchvision import datasets, transforms, models\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "#from tensorboard import SummaryWriter\n",
        "\n",
        "# number of subprocesses to use for data loading\n",
        "num_workers = 0\n",
        "# how many samples per batch to load\n",
        "batch_size = 10\n",
        "# percentage of training set to use as validation\n",
        "valid_size = 0.2\n",
        "# Data directory (should be changed according to your path)\n",
        "data_dir = 'gdrive/My Drive/Thesis/ModifiedSixClasses'\n",
        "\n",
        "# Define transforms for the training data and testing data\n",
        "train_transforms = transforms.Compose([transforms.RandomRotation(30),\n",
        "                                       transforms.RandomResizedCrop(224),\n",
        "                                       transforms.RandomHorizontalFlip(),\n",
        "                                       transforms.ToTensor(),\n",
        "                                       transforms.Normalize([0.485, 0.456, 0.406],\n",
        "                                                            [0.229, 0.224, 0.225])])\n",
        "\n",
        "test_transforms = transforms.Compose([transforms.Resize(224),\n",
        "                                      transforms.CenterCrop(224),\n",
        "                                      transforms.ToTensor(),\n",
        "                                      transforms.Normalize([0.485, 0.456, 0.406],\n",
        "                                                           [0.229, 0.224, 0.225])])\n",
        "# choose the training and test datasets (change the path according to your data) you can put the path directly instead of: data_dir+'Train' for the train images\n",
        "#or any other path to your images which you need to extract the features from\n",
        "train_data = datasets.ImageFolder(data_dir + '/Train', transform=train_transforms)\n",
        "#you can put the path directly instead of: data_dir+'Test' for the test images \n",
        "test_data = datasets.ImageFolder(data_dir + '/Test', transform=test_transforms)\n",
        "\n",
        "#splitting the data into train, validation and test sets\n",
        "# obtain training indices that will be used for validation\n",
        "num_train = len(train_data)\n",
        "indices = list(range(num_train))\n",
        "np.random.shuffle(indices)\n",
        "split = int(np.floor(valid_size * num_train))\n",
        "train_idx, valid_idx = indices[split:], indices[:split]\n",
        "test_idx = [i for i in range(len(test_data))]\n",
        "# define samplers for obtaining training and validation batches\n",
        "train_sampler = SubsetRandomSampler(train_idx)\n",
        "valid_sampler = SubsetRandomSampler(valid_idx)\n",
        "test_sampler = SubsetRandomSampler(test_idx)\n",
        "# prepare data loaders (combine dataset and sampler)\n",
        "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,\n",
        "    sampler=train_sampler, num_workers=num_workers)\n",
        "valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, \n",
        "    sampler=valid_sampler, num_workers=num_workers)\n",
        "test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, \n",
        "    sampler=test_sampler,drop_last=False,num_workers=num_workers)\n",
        "\n",
        "# specify the image classes\n",
        "classes = ['1','2','3' ,'4' ,'5' ,'6']\n",
        "\n",
        "print(\"Loaded the data successfully ...\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded the data successfully ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UeIiLdLD_mbl",
        "colab_type": "code",
        "outputId": "76de3b5a-8660-4cf4-b2c4-1440850766fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Using & Viewing pretrained model (we do not change anything on the model here yet)\n",
        "#example used model here is densenet121, you can change it to any other model \n",
        "print(\"Viewing the pretrained Model ...\")\n",
        "model = models.densenet121(pretrained=True)\n",
        "print(model)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Viewing the pretrained Model ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/densenet121-a639ec97.pth\" to /root/.cache/torch/checkpoints/densenet121-a639ec97.pth\n",
            "100%|██████████| 32342954/32342954 [00:01<00:00, 24275100.74it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "DenseNet(\n",
            "  (features): Sequential(\n",
            "    (conv0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
            "    (norm0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (relu0): ReLU(inplace)\n",
            "    (pool0): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
            "    (denseblock1): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(96, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(160, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(192, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(224, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(224, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (transition1): _Transition(\n",
            "      (norm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
            "    )\n",
            "    (denseblock2): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(160, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(192, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(224, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(224, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(288, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer7): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(320, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer8): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(352, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(352, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer9): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(384, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer10): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(416, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(416, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer11): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(448, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(448, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer12): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(480, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (transition2): _Transition(\n",
            "      (norm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
            "    )\n",
            "    (denseblock3): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(288, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(320, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(352, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(352, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(384, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(416, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(416, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer7): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(448, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(448, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer8): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(480, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer9): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer10): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(544, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(544, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer11): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(576, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer12): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(608, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(608, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer13): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(640, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer14): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(672, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer15): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(704, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer16): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(736, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(736, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer17): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer18): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(800, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(800, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer19): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(832, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer20): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(864, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer21): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(896, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(896, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer22): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(928, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(928, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer23): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(960, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer24): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(992, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(992, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (transition3): _Transition(\n",
            "      (norm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
            "    )\n",
            "    (denseblock4): _DenseBlock(\n",
            "      (denselayer1): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer2): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(544, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(544, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer3): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(576, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer4): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(608, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(608, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer5): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(640, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer6): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(672, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer7): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(704, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer8): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(736, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(736, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer9): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer10): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(800, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(800, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer11): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(832, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer12): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(864, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer13): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(896, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(896, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer14): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(928, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(928, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer15): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(960, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "      (denselayer16): _DenseLayer(\n",
            "        (norm1): BatchNorm2d(992, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu1): ReLU(inplace)\n",
            "        (conv1): Conv2d(992, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu2): ReLU(inplace)\n",
            "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      )\n",
            "    )\n",
            "    (norm5): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  )\n",
            "  (classifier): Linear(in_features=1024, out_features=1000, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bG4TR4-9TITh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOJXlwBGSgdy",
        "colab_type": "text"
      },
      "source": [
        "**have a look now at the last output, look at the last layer, it is named \"classifier\", usually this is the layer that you use to get the number of the classes. Now we need to replaced its output with the number of the features we want in the feature vector. For exmaple in the following I wanted to have a feature vector of size 1024 for each image, feel free to change this as you wish based on the comments. **"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Alsg8WS_poy",
        "colab_type": "code",
        "outputId": "8c6b0f82-5340-476d-a7a9-f89609537693",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "print(\"Viewing the modified Network Architecture ...\")\n",
        "\n",
        "# define the layer you want to replace, you can also find this part online in some resources as the \"identity\" layer\n",
        "class Classifier(nn.Module):\n",
        "    def __init__(self):\n",
        "       \n",
        "        super(Classifier, self).__init__()\n",
        "        ######important######\n",
        "        #change this line according to how big you want your feature vector to be, here the feature vector will have a size of 1024\n",
        "        #self.fc1= nn.Linear(first input depends on the last layer of the model before the \"classifier\" layer, or sometimes it is called \"fc\" layer, the output is the size of your feature vector)\n",
        "        self.fc1 = nn.Linear(1024, 1024) \n",
        "\n",
        "    def forward(self, x):\n",
        "        # make sure input tensor is flattened\n",
        "        x = x.view(x.shape[0], -1)\n",
        "        x = self.fc1(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "# create a complete CNN\n",
        "model.classifier = Classifier()\n",
        "#if the last layer in your model is called fc then use the following line\n",
        "#model.fc =  Classifier()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Viewing the Network Architecture ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'for param in model.parameters():\\n    param.requires_grad = False'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2o2crZMs_Sbj",
        "colab_type": "code",
        "outputId": "bc1baca8-9349-4e36-9f0d-c70b562c4c38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#to test the output of your  model (the feature vector size)\n",
        "x = torch.randn(1, 3, 224, 224)\n",
        "output = model(x)\n",
        "\n",
        "print(output.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 1024])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nKpUCofj2e0V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get the featuer vectors of all the images in the test set, if you want to extract the features in the train set replace \"test_loader\" by \"train_loader\"\n",
        "if train_on_gpu:\n",
        "    model.cuda()\n",
        "    \n",
        "print(\"Getting the feature vectors of your images is in Progress ...\")\n",
        "FV= [] #to collect all the feature vectors\n",
        "\n",
        "model.eval()\n",
        "# iterate over test data\n",
        "for data, target in test_loader:\n",
        "    # move tensors to GPU if CUDA is available\n",
        "    if train_on_gpu:\n",
        "        data, target = data.cuda(), target.cuda()\n",
        "    # forward pass: compute predicted outputs by passing inputs to the model\n",
        "    output = model(data)\n",
        "    FV.append(output)\n",
        "    #print(output.shape)\n",
        "print(\"Testing the Trained Network is Finished\")\n",
        "print(FV)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qpm4GGbn_6cT",
        "colab_type": "code",
        "outputId": "e8f61a1e-731c-42c1-d843-26805b97b195",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Get the featuer vectors of all the images in the test set, if you want to extract the features in the train set replace \"test_loader\" by \"train_loader\"\n",
        "print(\"Getting the feature vectors of your images is in Progress ...\")\n",
        "# Test the Trained Network on CPU\n",
        "labels= []\n",
        "FV= []  #to collect all the feature vectors\n",
        "counter= 0\n",
        "model.cpu()\n",
        "with torch.no_grad():\n",
        "  for i, (data, targets) in enumerate(train_loader):\n",
        "    output = model(data)\n",
        "    FV.append(output.data.tolist())\n",
        "    labels.append(targets.item())\n",
        "    counter= i\n",
        "    \n",
        "print(\"Getting the feature vectors of your images is Finished!\")\n",
        "print(\"Number of Images which we extracted features from is: \", counter)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing the Trained Network is Finished\n",
            "Number of Images 831\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oImOuuMhDSSa",
        "colab_type": "code",
        "outputId": "066534aa-3ca5-4232-fb42-9955444872f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "FV= np.array(FV) #convert the list into array to be able to reshape the feature vectors\n",
        "#print(FV)\n",
        "print(FV.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[ 0.32370925  0.82736659 -0.27542758 ...  0.43905091 -0.34853292\n",
            "    0.29555115]]\n",
            "\n",
            " [[ 0.34610209  0.83329141 -0.23297775 ...  0.4437449  -0.35099807\n",
            "    0.26116562]]\n",
            "\n",
            " [[ 0.25344753  0.79795212 -0.25761664 ...  0.39657754 -0.36041403\n",
            "    0.28936246]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[ 0.31986853  0.8511312  -0.25927624 ...  0.46423244 -0.37248254\n",
            "    0.29600784]]\n",
            "\n",
            " [[ 0.27990925  0.83055079 -0.24629395 ...  0.42959565 -0.35889852\n",
            "    0.29659656]]\n",
            "\n",
            " [[ 0.32579982  0.80333358 -0.275161   ...  0.47051948 -0.32189947\n",
            "    0.27233046]]]\n",
            "(832, 1, 1024)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A7_vyS66cwol",
        "colab_type": "code",
        "outputId": "d512c710-d856-4818-bb08-a12344255c74",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "FV2=  np.empty((1024, 832)) #choose the shape you need, here I reshaped it this way because I need it to be like this (length of the Feature vector, number of images)\n",
        "for i, val in enumerate(FV):\n",
        "  val= val.reshape(1024, 1)\n",
        "  np.concatenate((FV2, val), axis= 1)\n",
        " \n",
        "#print(FV2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 0.32370925  0.82736659 -0.27542758 ... -1.10647118 -0.2693184\n",
            "   0.9853121 ]\n",
            " [-0.02894734 -0.54835284  0.03568567 ... -0.29622668 -0.16395263\n",
            "  -0.49134821]\n",
            " [-0.28973013  0.31401974 -0.22905207 ...  0.27917466 -0.01399475\n",
            "  -0.18747541]\n",
            " ...\n",
            " [-0.10423304 -0.3947815   0.7056545  ... -0.44252345  0.34429666\n",
            "   0.5190084 ]\n",
            " [-0.13648178  0.0267427   0.33430025 ... -0.14720182 -1.03976405\n",
            "  -0.02464781]\n",
            " [-0.03570941  0.25726002 -0.19233489 ...  0.47051948 -0.32189947\n",
            "   0.27233046]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hPuwUgeXRWYk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# not necessary for you but for me I needed it as a data frame to creat a correlation matrix\n",
        "import pandas as pd\n",
        "df= pd.DataFrame(FV2, columns= labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NnHNr7w_Dop4",
        "colab_type": "code",
        "outputId": "3c3a9139-2d1f-48fa-ccc2-c6368a238b1c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(df)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "             2         1         2  ...         4         5         6\n",
            "0     0.323709  0.827367 -0.275428  ... -0.042195 -0.070360  0.007719\n",
            "1    -0.028947 -0.548353  0.035686  ... -0.040962 -0.027828 -0.105555\n",
            "2    -0.289730  0.314020 -0.229052  ... -0.010882  0.019849  0.008855\n",
            "3     0.226208 -0.211631 -0.460487  ... -0.064621 -0.131572  0.061088\n",
            "4    -0.042015 -0.488078  0.425623  ...  0.020321 -0.037796 -0.005671\n",
            "5    -0.233459  0.039377 -0.371909  ... -0.004114  0.018708 -0.087440\n",
            "6    -0.133548  0.480235 -0.922469  ... -0.011367 -0.020877 -0.111912\n",
            "7     0.057377 -0.266844 -0.850585  ... -0.074769 -0.032866 -0.036245\n",
            "8     0.222092  0.831864  0.302121  ... -0.013030 -0.046477 -0.145518\n",
            "9    -0.996068 -0.904400 -0.459541  ...  0.005383 -0.018453 -0.064317\n",
            "10    0.539975  0.031433  0.225427  ...  0.016733 -0.024616  0.055795\n",
            "11   -0.176606 -0.472893 -0.319264  ... -0.030013 -0.038252 -0.005578\n",
            "12    0.212455 -0.023769  0.460573  ...  0.020032 -0.012287  0.012153\n",
            "13   -0.112499 -0.410348  0.710049  ... -0.045581 -0.026397  0.095644\n",
            "14   -0.144165  0.064350  0.338348  ... -0.028415 -0.032002 -0.003967\n",
            "15   -0.050279  0.240759 -0.137928  ... -0.000705  0.097423  0.040087\n",
            "16    0.308758  0.859742 -0.271643  ... -0.043703 -0.067935  0.008193\n",
            "17   -0.072509 -0.574590  0.062330  ... -0.040582 -0.028805 -0.101339\n",
            "18   -0.316861  0.294934 -0.230545  ... -0.008057  0.023357  0.009471\n",
            "19    0.171530 -0.213468 -0.389711  ... -0.055574 -0.134135  0.066378\n",
            "20    0.018792 -0.533816  0.422000  ...  0.019920 -0.041896 -0.010909\n",
            "21   -0.211293 -0.002023 -0.349239  ... -0.005486  0.022555 -0.092924\n",
            "22   -0.181017  0.502456 -0.944683  ... -0.012322 -0.013309 -0.102813\n",
            "23    0.073244 -0.256838 -0.885268  ... -0.085446 -0.035791 -0.038115\n",
            "24    0.193966  0.853769  0.263922  ... -0.016545 -0.050435 -0.148553\n",
            "25   -0.928008 -0.841964 -0.461393  ...  0.009592 -0.011522 -0.075139\n",
            "26    0.524882  0.019441  0.221098  ...  0.013994 -0.028726  0.050198\n",
            "27   -0.175128 -0.455783 -0.322034  ... -0.032232 -0.034869 -0.008377\n",
            "28    0.283248  0.004079  0.507516  ...  0.020491 -0.014681  0.015814\n",
            "29   -0.088013 -0.412133  0.689064  ... -0.048827 -0.019014  0.108956\n",
            "...        ...       ...       ...  ...       ...       ...       ...\n",
            "994  -0.298781  0.290243 -0.242718  ... -0.014036  0.018941  0.009633\n",
            "995   0.228639 -0.237586 -0.487801  ... -0.057122 -0.134763  0.062016\n",
            "996  -0.043185 -0.530838  0.416535  ...  0.018067 -0.038849 -0.007210\n",
            "997  -0.213090 -0.012559 -0.304003  ... -0.002284  0.019858 -0.091937\n",
            "998  -0.156907  0.560482 -0.953550  ... -0.010104 -0.016121 -0.118572\n",
            "999   0.095814 -0.272148 -0.848706  ... -0.078203 -0.032744 -0.027153\n",
            "1000  0.242041  0.868178  0.310342  ... -0.013573 -0.051310 -0.147037\n",
            "1001 -0.984705 -0.838371 -0.464141  ...  0.009604 -0.016186 -0.063578\n",
            "1002  0.548061  0.041833  0.237495  ...  0.014182 -0.024934  0.053853\n",
            "1003 -0.216216 -0.429393 -0.353130  ... -0.029640 -0.034382  0.008033\n",
            "1004  0.269522 -0.037969  0.516438  ...  0.017740 -0.015239  0.001948\n",
            "1005 -0.138939 -0.367644  0.720588  ... -0.048777 -0.022738  0.107549\n",
            "1006 -0.135976  0.028310  0.359360  ... -0.032248 -0.032021  0.009312\n",
            "1007 -0.014698  0.234306 -0.174832  ...  0.001120  0.092392  0.035468\n",
            "1008  0.299917  0.857330 -0.281211  ... -0.036132 -0.068372  0.013328\n",
            "1009 -0.018036 -0.592765  0.036256  ... -0.040050 -0.025067 -0.103635\n",
            "1010 -0.270159  0.302356 -0.248081  ... -0.011271  0.020810  0.013078\n",
            "1011  0.198821 -0.218523 -0.468324  ... -0.063397 -0.132434  0.066738\n",
            "1012 -0.020021 -0.548873  0.377709  ...  0.019755 -0.041570 -0.000551\n",
            "1013 -0.251612  0.011271 -0.341338  ... -0.006335  0.019525 -0.096909\n",
            "1014 -0.202784  0.538090 -0.969262  ... -0.007233 -0.019908 -0.114244\n",
            "1015  0.027443 -0.233825 -0.856717  ... -0.079898 -0.039585 -0.035316\n",
            "1016  0.208774  0.854106  0.321958  ... -0.013582 -0.047785 -0.143905\n",
            "1017 -0.889511 -0.880809 -0.412877  ...  0.009573 -0.013498 -0.060228\n",
            "1018  0.498554  0.021184  0.192414  ...  0.020203 -0.028870  0.043010\n",
            "1019 -0.222517 -0.451757 -0.328806  ... -0.035254 -0.032063 -0.004711\n",
            "1020  0.241413 -0.016950  0.517853  ...  0.022035 -0.014092  0.004034\n",
            "1021 -0.104233 -0.394782  0.705655  ... -0.042730 -0.025797  0.105456\n",
            "1022 -0.136482  0.026743  0.334300  ... -0.030917 -0.028410 -0.003159\n",
            "1023 -0.035709  0.257260 -0.192335  ... -0.006029  0.092564  0.037686\n",
            "\n",
            "[1024 rows x 838 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}