Adapter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4ec18445",
   "metadata": {},
   "source": [
    "## Alireza Heidari\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a4854d",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6ebfc213",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alireza/anaconda3/envs/torch/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaForSequenceClassification\n",
    "import torch\n",
    "from transformers import AdamW\n",
    "from transformers import get_scheduler\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import AutoTokenizer\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1297c438",
   "metadata": {},
   "source": [
    "# Load The Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "83bf2dc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'lm_head.decoder.weight', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.5.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.9.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.4.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.0.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.8.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.1.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.2.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.6.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.5.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.10.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.7.output.my_adapter_out.fc1.bias', 'classifier.out_proj.bias', 'roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.9.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.11.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.8.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.6.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.11.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.1.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.4.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.3.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.9.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.1.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.7.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.10.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.2.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.0.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.8.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.10.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.7.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.5.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.11.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.11.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.3.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.2.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.3.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.7.output.my_adapter_out.fc1.weight', 'classifier.out_proj.weight', 'roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.0.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.2.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.5.output.my_adapter_out.fc2.bias', 'roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.0.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.6.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.4.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.4.output.my_adapter_out.fc2.weight', 'roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc1.bias', 'roberta.encoder.layer.8.output.my_adapter_out.fc2.bias', 'classifier.dense.bias', 'roberta.encoder.layer.3.output.my_adapter_out.fc2.bias', 'classifier.dense.weight', 'roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc2.bias', 'roberta.encoder.layer.1.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.9.output.my_adapter_out.fc1.bias', 'roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc1.weight', 'roberta.encoder.layer.6.output.my_adapter_out.fc1.weight', 'roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc2.weight', 'roberta.encoder.layer.10.output.my_adapter_out.fc1.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaForSequenceClassification(\n",
       "  (roberta): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): RobertaEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (1): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (2): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (3): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (4): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (5): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (6): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (7): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (8): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (9): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (10): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (11): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (my_adapter_self_out): MyAdapter(\n",
       "                (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "                (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "                (relu): ReLU()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (my_adapter_out): MyAdapter(\n",
       "              (fc1): Linear(in_features=768, out_features=64, bias=True)\n",
       "              (fc2): Linear(in_features=64, out_features=768, bias=True)\n",
       "              (relu): ReLU()\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): RobertaClassificationHead(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (out_proj): Linear(in_features=768, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=2)\n",
    "tokenizer = AutoTokenizer.from_pretrained('roberta-base')\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "806efa6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "    \n",
    "for name, param in model.named_parameters():\n",
    "    if \"my_adapter\" in name:\n",
    "        param.requires_grad = True\n",
    "\n",
    "for param in model.classifier.parameters():\n",
    "    param.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8fe125bc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.0.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.0.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.0.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.0.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.0.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.1.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.1.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.1.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.1.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.1.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.2.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.2.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.2.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.2.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.2.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.3.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.3.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.3.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.3.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.3.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.4.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.4.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.4.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.4.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.4.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.5.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.5.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.5.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.5.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.5.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.6.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.6.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.6.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.6.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.6.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.7.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.7.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.7.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.7.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.7.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.8.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.8.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.8.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.8.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.8.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.9.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.9.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.9.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.9.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.9.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.10.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.10.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.10.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.10.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.10.output.my_adapter_out.fc2.bias\n",
      "roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc1.weight\n",
      "roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc1.bias\n",
      "roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc2.weight\n",
      "roberta.encoder.layer.11.attention.output.my_adapter_self_out.fc2.bias\n",
      "roberta.encoder.layer.11.output.my_adapter_out.fc1.weight\n",
      "roberta.encoder.layer.11.output.my_adapter_out.fc1.bias\n",
      "roberta.encoder.layer.11.output.my_adapter_out.fc2.weight\n",
      "roberta.encoder.layer.11.output.my_adapter_out.fc2.bias\n",
      "classifier.dense.weight\n",
      "classifier.dense.bias\n",
      "classifier.out_proj.weight\n",
      "classifier.out_proj.bias\n"
     ]
    }
   ],
   "source": [
    "for name, param in model.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "141bed9c",
   "metadata": {},
   "source": [
    "# Prepare Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f1ce2984",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset imdb (/home/alireza/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0)\n",
      "100%|████████████████████████████████████████████| 3/3 [00:00<00:00, 473.54it/s]\n"
     ]
    }
   ],
   "source": [
    "class IMDBDataset(Dataset):\n",
    "    def __init__(self, dataset, tokenizer):\n",
    "        self.dataset = dataset\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        text = self.dataset[idx]['text']\n",
    "        label = self.dataset[idx]['label']\n",
    "        encoding = self.tokenizer(text, truncation=True, padding='max_length', max_length=256, return_tensors='pt')\n",
    "        return {\n",
    "            'input_ids': encoding['input_ids'].squeeze(),\n",
    "            'attention_mask': encoding['attention_mask'].squeeze(),\n",
    "            'labels': torch.tensor(label, dtype=torch.long)\n",
    "        }\n",
    "\n",
    "imdb_dataset = load_dataset(\"imdb\")\n",
    "\n",
    "train_dataset = IMDBDataset(imdb_dataset['train'], tokenizer)\n",
    "val_dataset = IMDBDataset(imdb_dataset['test'], tokenizer)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=12, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "88fc7ef3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[    0,   100,   802,  ...,     1,     1,     1],\n",
      "        [    0,   113, 42362,  ...,     1,     1,     1],\n",
      "        [    0,   100,   206,  ...,     1,     1,     1],\n",
      "        ...,\n",
      "        [    0,   113,   243,  ...,     1,     1,     1],\n",
      "        [    0,   108, 34673,  ...,  4889,   149,     2],\n",
      "        [    0,  7516,   272,  ...,   734,  2497,     2]]), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 1, 1, 1],\n",
      "        [1, 1, 1,  ..., 1, 1, 1]]), 'labels': tensor([1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0])}\n"
     ]
    }
   ],
   "source": [
    "for batch in train_loader:\n",
    "    print(batch)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff56f157",
   "metadata": {},
   "source": [
    "# Set Device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "df6761b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "model.to(device)\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "909ada5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Wed May  3 19:03:32 2023       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 510.108.03   Driver Version: 510.108.03   CUDA Version: 11.6     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0 Off |                  N/A |\r\n",
      "| N/A   54C    P0     8W /  N/A |   1017MiB /  4096MiB |     22%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|    0   N/A  N/A      2097      G   /usr/lib/xorg/Xorg                  4MiB |\r\n",
      "|    0   N/A  N/A     20046      C   ...da3/envs/torch/bin/python     1009MiB |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f383a97",
   "metadata": {},
   "source": [
    "# Set Optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8c3591da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20840\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alireza/Desktop/SUT/Semester8/DeepLearning/HW/HW3/transformers/src/transformers/optimization.py:407: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "optimizer = AdamW(model.parameters(), lr=3e-4)\n",
    "\n",
    "num_epochs = 10\n",
    "num_training_steps = num_epochs * len(train_loader)\n",
    "lr_scheduler = get_scheduler(\n",
    "    \"linear\",\n",
    "    optimizer=optimizer,\n",
    "    num_warmup_steps=0,\n",
    "    num_training_steps=num_training_steps,\n",
    ")\n",
    "print(num_training_steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3392c911",
   "metadata": {},
   "source": [
    "# Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e23f23a5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|███▌                                | 2080/20840 [14:28<2:10:24,  2.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.22611150621476736, epoch = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|███████▏                            | 4160/20840 [28:59<1:56:17,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.16857459795498234, epoch = 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██████████▊                         | 6240/20840 [43:30<1:41:38,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.13587871901953383, epoch = 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|██████████████▎                     | 8320/20840 [58:03<1:27:28,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.09862852213051107, epoch = 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|████████████████▍                | 10400/20840 [1:12:38<1:13:04,  2.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.07395832287454732, epoch = 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|████████████████████▉              | 12480/20840 [1:27:14<58:29,  2.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.04813371616686757, epoch = 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|████████████████████████▍          | 14560/20840 [1:41:47<43:46,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.03535903479497181, epoch = 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████▉       | 16640/20840 [1:56:19<29:16,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.02554674058588952, epoch = 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████▍   | 18720/20840 [2:10:50<14:48,  2.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.017479574276099197, epoch = 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████▉| 20800/20840 [2:25:22<00:16,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.011762766072365212, epoch = 9\n"
     ]
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "progress_bar = tqdm(range(num_training_steps))\n",
    "\n",
    "model.train()\n",
    "for epoch in range(num_epochs):\n",
    "    epoch_loss = 0\n",
    "    i = 1\n",
    "    for batch in train_loader:\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        outputs = model(**batch)\n",
    "        loss = outputs.loss\n",
    "        loss.backward()\n",
    "        epoch_loss += loss.item()\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "        if i % 10 == 0:\n",
    "            progress_bar.update(10)\n",
    "        i += 1 \n",
    "        \n",
    "    print(f'loss = {epoch_loss / i}, epoch = {epoch}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efeb345e",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a180e4e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.94156,\n",
       " 'f1': 0.9417812313209802,\n",
       " 'precision': 0.9382294561333863,\n",
       " 'recall': 0.94536}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "metric = evaluate.combine([\"accuracy\", \"f1\", \"precision\", \"recall\"])\n",
    "model.eval()\n",
    "\n",
    "for batch in val_loader:\n",
    "    batch = {k: v.to(device) for k, v in batch.items()}\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**batch)\n",
    "\n",
    "    logits = outputs.logits\n",
    "    predictions = torch.argmax(logits, dim=-1)\n",
    "    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n",
    "\n",
    "metric.compute()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}