38-Finetuning-Bertweet-on-CMV-dataset.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"38-Finetuning-Bertweet-on-CMV-dataset.ipynb","private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyO2mOR10NcyKt641K9tWpjB"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","source":["__Objective__: To fine-tune Bertweet model using CMV dataset and save it in `MyDrive/DL/models/bertweet`\n","\n","__Runtime__: GPU"],"metadata":{"id":"EMBNLYWn9Wxj"}},{"cell_type":"code","source":["!nvidia-smi"],"metadata":{"id":"q_xOUXRcBxME"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"id":"eb_gXGMM9LUj"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"code","source":["import pandas as pd\n","import numpy as np\n","from tqdm import tqdm \n","import pickle as pkl\n","import matplotlib.pyplot as plt\n","from matplotlib import colors"],"metadata":{"id":"0tTBiThK92Q_"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Training Bertweet using CMV dataset"],"metadata":{"id":"poyj2pmQ-IEp"}},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"id":"fHe7o485-bBd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import torch\n","from transformers import RobertaForSequenceClassification, AutoTokenizer\n","from torch.utils.data import DataLoader\n","from transformers import AdamW"],"metadata":{"id":"QuYgETd9-vOu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import os\n","\n","def read_split(dir):\n","    texts = []\n","    labels = []\n","    with open(dir, 'r', encoding='utf-8') as f:\n","        lines = f.readlines()[1:]\n","        for line in lines:\n","            line = line.strip().split(\",\")\n","            assert(len(line) == 2)\n","            label = line[0].strip()\n","            text = line[1].strip()\n","            texts.append(text)\n","            if label == 'AH':\n","                labels.append(1)\n","            else:\n","                labels.append(0)\n","    return texts, labels\n","\n","\n","train_texts, train_labels = read_split('/content/gdrive/MyDrive/DL/dataset/pytorch/train.csv')\n","test_texts, test_labels = read_split('/content/gdrive/MyDrive/DL/dataset/pytorch/test.csv')"],"metadata":{"id":"pwHuNkny-GmJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)"],"metadata":{"id":"RJm-RmOj-TCr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["max_seq_length = 64\n","train_encodings = tokenizer(train_texts, truncation=True, max_length=max_seq_length, padding=\"max_length\")\n","test_encodings = tokenizer(test_texts, truncation=True, max_length=max_seq_length, padding=\"max_length\")"],"metadata":{"id":"RDjbUoWC-5fp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class CustomDataset(torch.utils.data.Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","\n","    def __len__(self):\n","        return len(self.labels)\n","\n","train_dataset = CustomDataset(train_encodings, train_labels)\n","test_dataset = CustomDataset(test_encodings, test_labels)"],"metadata":{"id":"LbM9v6sH_C0Q"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"],"metadata":{"id":"aalbtnnV_HmZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["model = RobertaForSequenceClassification.from_pretrained(\"vinai/bertweet-base\")\n","model.to(device)\n","model.train()"],"metadata":{"id":"V5qAz-V7_RER"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n","optim = AdamW(model.parameters(), lr=5e-5)"],"metadata":{"id":"egSoTYa9_X1N"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from tqdm import tqdm\n","for epoch in range(3):\n","    for batch in tqdm(train_loader):\n","        optim.zero_grad()\n","        input_ids = batch['input_ids'].to(device)\n","        attention_mask = batch['attention_mask'].to(device)\n","        labels = batch['labels'].to(device)\n","        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n","        loss = outputs[0]\n","        loss.backward()\n","        optim.step()"],"metadata":{"id":"R3WFBQkz_q3U"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","!pip install datasets\n","from datasets import load_metric"],"metadata":{"id":"MQmkA23C_yfp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["metric= load_metric(\"accuracy\")\n","model.eval()\n","eval_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n","for batch in eval_loader:\n","    batch = {k: v.to(device) for k, v in batch.items()}\n","    with torch.no_grad():\n","        outputs = model(**batch)\n","\n","    logits = outputs.logits\n","    predictions = torch.argmax(logits, dim=-1)\n","    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n","\n","metric.compute()"],"metadata":{"id":"MMc6jbuQFtIx"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n","\n","output_dir = '/content/gdrive/MyDrive/DL/models/bertweet'\n","\n","# Create output directory if needed\n","if not os.path.exists(output_dir):\n","    os.makedirs(output_dir)\n","\n","print(\"Saving model to %s\" % output_dir)\n","\n","# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n","# They can then be reloaded using `from_pretrained()`\n","model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n","model_to_save.save_pretrained(output_dir)\n","tokenizer.save_pretrained(output_dir)"],"metadata":{"id":"QntePHGNF2T2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"id":"VLtbVu37GUfg"},"execution_count":null,"outputs":[]}]}