model_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hybrid Zero Shot Learning Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Loading Pre-Trained Models</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Loading BERT4RE</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaModel were not initialized from the model checkpoint at thearod5/bert4re and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BERT4RE loaded successfully.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModel\n",
    "\n",
    "# Specify the model name for BERT4RE\n",
    "model_name = \"thearod5/bert4re\"  # Replace with the correct model name or path if needed\n",
    "\n",
    "# Load the tokenizer and model\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "bert4re = AutoModel.from_pretrained(model_name)\n",
    "\n",
    "print(\"BERT4RE loaded successfully.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Testing BERT4RE</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding shape: torch.Size([1, 768])\n",
      "Sentence embedding: tensor([[-6.8532e-01, -2.0724e-01,  4.6072e-01,  3.8424e-01, -8.1794e-01,\n",
      "          6.2162e-02,  1.6305e-01,  6.3627e-01, -6.2024e-01,  2.3580e-01,\n",
      "         -4.1779e-01,  4.5084e-02,  1.0638e+00, -5.3630e-02, -6.2707e-01,\n",
      "         -2.7198e-01,  5.0179e-01,  6.1894e-01, -5.1897e-01, -5.8201e-01,\n",
      "          4.4262e-01, -2.5631e-01,  1.5594e-01, -1.6746e-01, -7.8109e-01,\n",
      "         -2.5813e-01, -9.9017e-01, -8.3839e-01, -4.7253e-01, -7.3076e-01,\n",
      "          1.2344e-01,  6.4781e-01, -3.2000e-01,  1.6285e-01, -9.6721e-01,\n",
      "          8.0904e-01, -4.3488e-01, -5.2084e-01, -3.0309e-01, -2.9167e-01,\n",
      "         -2.0697e-01,  2.6938e-01,  6.5628e-01, -6.7765e-01, -3.0446e-01,\n",
      "          1.2397e+00,  4.9944e-01,  7.6995e-01, -2.0611e-01,  4.7138e-01,\n",
      "         -1.2173e-01,  2.2105e-01, -1.7483e-01, -3.7982e-01, -4.4242e-01,\n",
      "          2.8855e-01, -2.1307e-01,  2.2410e-01, -8.2249e-01, -1.9172e-01,\n",
      "          6.2317e-02, -1.4458e-01, -6.6763e-01, -4.3271e-01,  6.2310e-01,\n",
      "          1.0593e+00, -1.9047e-01, -8.8285e-01,  3.9643e-01, -2.2702e-01,\n",
      "         -1.1167e-01,  6.9354e-01,  2.7095e-01, -1.3017e-01,  8.2746e-01,\n",
      "          3.8635e-01,  2.7911e-01,  5.3400e-01, -3.9682e-01,  1.4226e-01,\n",
      "         -1.8565e-04,  4.1742e-01, -1.8472e-01, -9.1909e-02,  1.4030e-01,\n",
      "         -5.5507e-02, -2.8642e-01, -8.3176e-01,  3.7613e-01,  1.6504e-01,\n",
      "          1.1727e-01, -8.6636e-01, -2.3921e-01, -9.0296e-01, -1.6369e-01,\n",
      "         -2.9436e-01,  6.1390e-01,  2.7809e-01,  4.7365e-01,  5.5475e-01,\n",
      "         -1.6866e-01,  3.8195e-01, -2.9224e-01,  4.5290e-01,  5.2320e-01,\n",
      "         -6.0292e-01, -6.4288e-01,  6.0261e-01,  8.1288e-01,  6.6173e-01,\n",
      "          1.3340e-01,  1.9146e-01,  7.0710e-01,  4.1730e-01, -7.6968e-01,\n",
      "          9.9758e-01,  1.0313e+00,  1.9288e-01,  4.5088e-02, -5.6751e-03,\n",
      "         -9.0228e-01, -7.4607e-02,  5.0658e-01,  7.6038e-01,  6.3960e-01,\n",
      "          5.4057e-01,  1.5826e-01,  2.4927e-01, -8.1024e-01, -5.8266e-01,\n",
      "          1.8387e-01, -4.2507e-01, -1.5630e-01,  8.1115e-01,  5.0855e-01,\n",
      "         -6.9953e-02,  8.3310e-01, -3.8837e-02,  2.8982e-01,  6.0805e-02,\n",
      "         -1.1833e-01,  7.7890e-01,  1.3937e-01, -6.5588e-01,  5.8089e-01,\n",
      "          1.6419e-01, -1.1383e+00, -4.4042e-01, -3.9442e-01,  6.4422e-01,\n",
      "          3.5062e-01, -1.1031e-02,  3.0271e-02,  7.8850e-01,  1.3924e-02,\n",
      "         -3.8295e-01, -1.6856e-01,  4.3517e-02,  1.7534e-01, -4.7677e-01,\n",
      "         -6.2093e-01, -1.1636e-01,  4.3566e-01, -1.1074e+00, -6.4052e-01,\n",
      "         -2.2562e-01, -2.0804e-01, -1.1711e-01, -8.2458e-02,  1.3404e-01,\n",
      "         -2.7735e-01,  3.8134e-01, -2.6244e-02, -1.4318e-01, -3.5825e-02,\n",
      "         -7.9087e-01,  1.8469e-01,  9.8688e-02, -7.4105e-01,  8.1201e-01,\n",
      "          3.9581e-01, -2.6474e-01,  1.1060e+00, -2.2073e-01,  2.8011e-01,\n",
      "         -5.4344e-01,  6.9092e-01, -5.5831e-01,  4.5330e-01,  1.1604e+00,\n",
      "         -1.1912e+00, -6.7060e-01, -7.1368e-02,  1.2014e+00, -4.7757e-01,\n",
      "         -8.4111e-01, -1.0078e+00, -3.5834e-01, -3.3932e-01, -3.4293e-01,\n",
      "          1.4208e-01, -3.4625e-01,  4.1799e-01, -5.5067e-01,  8.5032e-01,\n",
      "         -5.1402e-01,  1.6856e-03, -7.2520e-01,  4.0262e-01,  9.6769e-01,\n",
      "         -2.6986e-01, -5.0868e-02, -9.2273e-01, -3.9906e-01, -8.7213e-01,\n",
      "         -8.8527e-02, -4.1587e-02,  2.1507e-01,  6.5836e-01,  2.4835e-01,\n",
      "         -6.2142e-02, -3.2046e-02,  1.2926e-02, -1.1674e-01, -2.0020e-01,\n",
      "         -1.6695e-01,  5.6347e-01,  5.4708e-01,  2.1452e-01,  5.0244e-02,\n",
      "          6.1592e-01,  4.8768e-01, -9.8797e-01, -2.4205e-01,  2.9072e-01,\n",
      "          3.0770e-01,  7.1053e-01, -3.1988e-01,  5.1111e-03, -5.7783e-01,\n",
      "         -2.7944e-01,  1.4805e-01, -3.4538e-01,  4.4893e-01,  3.3988e-01,\n",
      "          6.2897e-01, -7.5913e-02, -4.2137e-01,  7.6963e-02, -2.5660e-02,\n",
      "          8.1784e-01, -3.1772e-01,  3.1829e-01, -3.1442e-01, -9.4409e-02,\n",
      "          8.2942e-01,  6.6146e-01,  5.3704e-01,  6.2594e-02, -4.2839e-01,\n",
      "         -4.0627e-01, -5.3064e-01,  1.3463e+00,  8.2859e-01,  2.1157e-01,\n",
      "          1.3720e-01,  5.9076e-01,  9.8303e-02,  3.1763e-01, -1.9876e-01,\n",
      "         -2.3615e-02,  8.7503e-01, -1.4377e+00, -2.6334e-01,  6.0217e-03,\n",
      "          1.0624e-01,  1.0745e-01, -9.4083e-01,  1.4546e-01,  4.9799e-02,\n",
      "          7.8086e-01,  2.6523e-01,  1.2123e-01, -2.0607e-01,  1.7949e-01,\n",
      "          5.4491e-01, -3.0609e-01, -3.9934e-01, -4.1903e-01, -4.7874e-01,\n",
      "         -5.5051e-01, -5.8623e-01, -1.0223e+00,  1.8468e-02,  5.3008e-02,\n",
      "         -2.7325e-01,  2.5617e-01,  6.3387e-01, -2.9547e-01,  3.0782e-01,\n",
      "          3.7719e-01, -2.4868e-01,  1.1920e+00,  3.8098e-01, -5.5692e-01,\n",
      "         -3.3682e-02, -6.5796e-01,  1.0843e+00, -7.2933e-01,  6.9394e-01,\n",
      "          1.8125e-01, -6.6863e-01,  4.6582e-01,  1.9492e-01,  2.2605e-01,\n",
      "          4.1925e-01, -1.1345e-01, -6.7038e-01,  1.1599e+00, -9.5812e-01,\n",
      "         -5.5627e-01,  2.9258e-01, -6.7161e-01, -9.2611e-01, -6.5818e-01,\n",
      "          9.1941e-01,  3.5161e-02, -3.3327e-01,  2.8079e-01,  4.3820e-01,\n",
      "         -1.2309e-01,  5.8713e-01, -8.3699e-02,  7.7824e-02,  2.7092e-01,\n",
      "         -4.6283e-01, -1.5860e-01,  2.0651e-01,  2.1146e-01, -4.2294e-01,\n",
      "          3.5668e-02, -3.1109e-01,  3.8680e-01, -5.2347e-01,  3.7987e-01,\n",
      "          4.1101e-01, -7.6451e-01,  7.6301e-01,  3.1153e-01, -2.3975e-01,\n",
      "          2.1211e-02, -9.5925e-02, -6.6464e-02,  2.9365e-01,  2.7143e-01,\n",
      "          2.1230e-01, -6.8150e-01, -5.7189e-01, -7.9067e-01, -5.3492e-01,\n",
      "          5.1808e-01, -3.6346e-01, -2.6296e-01, -3.8321e-01, -6.5524e-02,\n",
      "          5.3479e-01,  7.1867e-01, -9.4390e-01, -2.7857e-01,  2.9756e-01,\n",
      "         -1.7465e-01, -6.0517e-02,  4.5423e-01,  2.3849e-01,  2.5983e-01,\n",
      "         -5.1371e-01, -5.7792e-01, -1.3192e+00, -2.8762e-02,  8.1293e-03,\n",
      "         -9.0155e-01,  6.4713e-02,  6.2222e-01, -7.5632e-01,  9.2442e-01,\n",
      "          3.6460e-01,  4.6314e-01,  6.7186e-01, -1.3432e-01, -4.8048e-01,\n",
      "         -1.4950e-01, -8.1986e-02, -1.0491e+00, -2.8088e-01, -5.3604e-01,\n",
      "          9.7242e-01,  4.2654e-02, -2.3766e-01,  2.1188e-01, -2.9622e-01,\n",
      "         -2.1772e-01, -4.7801e-01, -7.0881e-01,  7.1142e-02,  2.2492e-01,\n",
      "         -2.3028e-01,  1.5430e-01,  7.9595e-01,  4.2175e-01,  2.6606e-01,\n",
      "         -1.9890e-01,  7.3155e-02, -1.2705e+00,  8.9223e-01, -2.5985e-01,\n",
      "         -5.3011e-01,  2.1314e-01, -2.7442e-01,  7.2115e-01, -2.5030e-01,\n",
      "         -6.4411e-02,  5.4054e-01,  1.0327e+00, -2.7692e-01,  3.3275e-02,\n",
      "         -7.9071e-01,  3.0014e-02, -1.7233e-01,  2.5708e-01,  4.8519e-01,\n",
      "          5.9897e-01,  1.0385e+00, -4.6338e-01, -1.0798e-01, -4.5937e-01,\n",
      "         -4.9046e-01, -7.7939e-01, -8.2879e-01,  6.2846e-01, -2.0386e-01,\n",
      "         -2.5711e-02, -7.3078e-01, -3.4546e-01, -3.0867e-01, -2.8541e-01,\n",
      "         -3.6524e-01,  5.5988e-01, -3.6818e-01, -2.5498e-01,  1.1268e+00,\n",
      "          2.3857e-01, -3.6069e-01, -5.7243e-01, -7.4878e-01,  8.0146e-01,\n",
      "          1.0394e+00,  9.5758e-01, -2.9713e-01,  9.5443e-01, -4.2703e-01,\n",
      "          4.4781e-01, -3.3310e-01, -3.3864e-01,  3.9000e-02,  2.2729e-01,\n",
      "         -4.1833e-01, -1.1916e+00,  5.4031e-01, -1.3009e-01,  7.3292e-01,\n",
      "          7.9498e-02,  2.5522e-01,  3.3772e-01, -4.7213e-01,  4.5586e-01,\n",
      "          7.1821e-01, -4.4907e-01,  1.1906e-02, -4.6130e-01, -3.6034e-01,\n",
      "         -6.4408e-01, -5.0940e-01, -3.1065e-01,  4.2855e-01, -8.3412e-03,\n",
      "         -1.1051e+00, -7.5176e-01,  4.3736e-01,  7.8880e-01, -3.1291e-01,\n",
      "         -4.6355e-01, -6.8466e-02,  6.9275e-02, -3.4856e-02,  2.7648e-01,\n",
      "         -1.6952e-01, -2.3316e-01, -4.2244e-02, -1.8563e-01,  1.2102e-01,\n",
      "          2.0360e-01, -3.7737e-01, -4.2354e-01, -4.4591e-01, -5.7240e-01,\n",
      "          1.3304e-01,  4.1409e-01,  1.6811e-02,  9.7820e-01,  1.3374e-01,\n",
      "          4.4549e-01, -8.5734e-02,  1.1344e-01, -3.1202e-01, -1.0110e+00,\n",
      "          6.3114e-01, -2.4388e-05, -2.4939e-01, -8.2066e-02, -2.0002e-01,\n",
      "          1.0707e+00,  1.3371e-01, -1.1124e-01, -1.0582e+00,  9.8471e-01,\n",
      "         -6.2308e-01,  1.2456e+00, -1.3041e-01,  6.2524e-01,  5.5541e-01,\n",
      "          2.3887e-01,  1.9968e-01, -1.2403e-01, -6.6441e-01, -1.7109e-01,\n",
      "         -3.8607e-01, -5.3369e-01,  2.2026e-01, -4.5942e-01, -4.4989e-01,\n",
      "          5.1819e-01, -1.0899e-01,  6.1828e-01, -6.7276e-04,  6.7400e-01,\n",
      "          3.1368e-01, -2.8545e-01, -1.6577e-01,  1.7993e-01,  5.2502e-01,\n",
      "          3.2333e-01,  2.9797e-01, -1.8329e-01, -1.1352e-01, -1.0453e-02,\n",
      "          2.7900e-01, -8.0959e-01,  4.2945e-01,  4.2833e-01, -6.6381e-01,\n",
      "         -5.0178e-01,  8.9121e-02, -6.6942e-01, -3.2017e-01,  3.4237e-01,\n",
      "          1.8428e-01,  5.0199e-01,  2.7363e-02,  3.8963e-01, -7.6104e-03,\n",
      "          6.7474e-01,  4.2430e-01, -7.5814e-01,  4.9188e-02, -3.7803e-01,\n",
      "          3.6169e-01,  2.7371e-01,  2.7085e-01, -5.4630e-01, -5.9266e-01,\n",
      "         -4.6613e-01, -7.8505e-01, -7.9490e-01,  2.6156e-01, -4.7670e-01,\n",
      "          7.3507e-01, -3.3840e-03, -6.5934e-01, -5.3862e-02,  7.7431e-01,\n",
      "          3.8649e-02,  1.1570e+00, -2.4203e-01, -2.8430e-01,  1.1605e-01,\n",
      "          7.8543e-01, -7.2294e-02, -4.2563e-01,  7.4858e-02, -5.0143e-01,\n",
      "          7.5709e-01,  8.6086e-01, -1.5490e-02,  5.4941e-01, -2.1215e-01,\n",
      "         -2.4307e-01,  3.4003e-01, -3.4218e-01,  6.7429e-01,  2.8147e-01,\n",
      "         -1.2046e-01, -3.6585e-01,  1.5112e-01, -4.0874e-01,  6.5823e-01,\n",
      "         -1.1189e+00, -1.4758e-02, -8.1077e-01, -2.0344e-01,  3.8867e-01,\n",
      "          8.2286e-02, -1.5909e-01,  2.8545e-01,  2.5085e-01, -3.8983e-01,\n",
      "          2.3105e-01,  6.9935e-01,  9.6897e-01,  6.7185e-01,  1.5942e-01,\n",
      "         -7.5879e-02, -6.5723e-01, -4.8930e-01,  3.5455e-02,  2.4811e-01,\n",
      "         -3.8647e-01, -7.3703e-02, -6.0234e-01,  6.9979e-01,  2.9264e-01,\n",
      "          5.6714e-01, -9.8941e-01, -2.6718e-01, -2.5571e-01, -4.3028e-01,\n",
      "          1.0654e+00,  1.7164e-01, -3.0944e-01, -2.6888e-01, -1.3182e-01,\n",
      "          1.0800e+00,  3.8248e-01,  8.2244e-02,  1.7396e-01, -1.3422e-01,\n",
      "          3.5192e-01,  6.5641e-01, -7.0932e-01, -5.1174e-01, -2.9833e-01,\n",
      "         -1.0221e+00,  1.9474e-01,  3.7682e-01, -4.7817e-01, -2.6166e-01,\n",
      "         -7.4314e-02,  7.8024e-01,  4.7655e-01, -2.5932e-02,  4.8977e-01,\n",
      "          8.2411e-01,  3.2459e-02, -1.6905e-02,  1.9304e-01,  1.9627e-01,\n",
      "         -3.8339e-01,  1.6820e-01,  7.5627e-02,  1.8005e-01,  6.1852e-01,\n",
      "          7.1152e-01, -6.5393e-01, -9.6121e-01, -6.0294e-01, -2.8285e-01,\n",
      "         -6.5534e-01,  1.2622e-02,  2.7795e-01, -5.5061e-01,  3.0198e-01,\n",
      "         -4.1715e-01,  4.0265e-01, -7.2359e-01,  8.9875e-01,  1.5829e-01,\n",
      "          5.1829e-01,  6.0207e-01, -2.6154e-01,  2.1165e-01, -9.1675e-01,\n",
      "         -6.4031e-01,  3.0323e-01,  2.5849e-01,  9.7195e-02,  1.9886e-01,\n",
      "         -1.5753e-01,  1.6652e-01,  4.0687e-01,  4.9338e-01,  4.7455e-01,\n",
      "          8.0216e-01, -1.3669e-01, -4.1225e-01,  1.3540e-01,  2.1907e-01,\n",
      "          3.4964e-01,  1.7677e-01,  3.5716e-01,  5.4289e-01,  1.2464e+00,\n",
      "         -1.0710e+00, -4.4837e-01,  9.8286e-02, -1.9693e-01, -2.1467e-01,\n",
      "          1.3609e+00, -7.8953e-01,  2.7787e-01, -5.3765e-01, -1.7712e-01,\n",
      "          3.6436e-01, -4.6566e-01, -3.7990e-01,  2.9500e-01, -1.9579e-01,\n",
      "         -1.3658e-02, -8.0957e-01,  9.7304e-02, -3.2537e-02,  4.3492e-01,\n",
      "         -5.2883e-01,  7.9383e-01, -6.2554e-01, -3.3606e-01, -9.9928e-02,\n",
      "          1.1881e-01,  2.4333e-01,  6.4661e-01, -1.6491e-01, -5.1448e-01,\n",
      "         -1.0167e+00,  5.5928e-01, -1.9867e-01,  2.9733e-01,  1.9416e-01,\n",
      "         -7.8888e-01, -1.0514e+00,  5.6060e-01,  3.8174e-01,  3.2378e-01,\n",
      "         -3.2369e-01, -1.0238e+00,  6.8724e-01,  2.7558e-01, -1.6291e-01,\n",
      "         -8.6108e-02, -4.7421e-01,  5.2656e-01]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# Move model to CPU or GPU if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "bert4re.to(device)\n",
    "\n",
    "# Prepare a sample sentence and encode it\n",
    "input_text = \"John works at OpenAI as a researcher.\"\n",
    "inputs = tokenizer(input_text, return_tensors=\"pt\", padding=True, truncation=True, max_length=512).to(device)\n",
    "\n",
    "# Pass the input through the model\n",
    "with torch.no_grad():  # Disable gradient calculation for inference\n",
    "    outputs = bert4re(**inputs)\n",
    "\n",
    "# Extract the embeddings (hidden states of the last layer)\n",
    "embeddings = outputs.last_hidden_state  # Shape: (batch_size, sequence_length, hidden_size)\n",
    "\n",
    "# Optionally, get sentence-level embedding by averaging token embeddings (ignoring padding tokens)\n",
    "sentence_embedding = embeddings.mean(dim=1)  # Shape: (batch_size, hidden_size)\n",
    "\n",
    "# Print the output shape and sentence embedding\n",
    "print(\"Embedding shape:\", sentence_embedding.shape)\n",
    "print(\"Sentence embedding:\", sentence_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Loading SBERT</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SBERT model loaded successfully.\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "\n",
    "# Load a pretrained SBERT model\n",
    "model_name = \"bert-base-nli-mean-tokens\" # Compatible with 768 Dimensions to match BERT4RE\n",
    "sbert = SentenceTransformer(model_name)\n",
    "\n",
    "print(\"SBERT model loaded successfully.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Testing SBERT</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding shape: (2, 768)\n",
      "Embeddings: [[ 0.0507936  -0.02515819  1.4253851  ... -0.79881257 -0.81836474\n",
      "   0.15679334]\n",
      " [-0.319343    0.04131207  1.000955   ... -0.6872918  -1.4411713\n",
      "   0.02773192]]\n"
     ]
    }
   ],
   "source": [
    "# Example sentences to encode\n",
    "sentences = [\"This is a test sentence.\", \"Sentence embeddings are useful for NLP tasks.\"]\n",
    "\n",
    "# Generate sentence embeddings\n",
    "embeddings = sbert.encode(sentences)\n",
    "\n",
    "# Print the shape and embeddings\n",
    "print(\"Embedding shape:\", embeddings.shape)\n",
    "print(\"Embeddings:\", embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Data Preprocessing</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Data Retrieval</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INPUT</th>\n",
       "      <th>TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The system shall refresh the display every 60 ...</td>\n",
       "      <td>PE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The application shall match the color of the s...</td>\n",
       "      <td>LF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If projected  the data must be readable.  On a...</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The product shall be available during normal b...</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>If projected  the data must be understandable....</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               INPUT TYPE\n",
       "0  The system shall refresh the display every 60 ...   PE\n",
       "1  The application shall match the color of the s...   LF\n",
       "2  If projected  the data must be readable.  On a...   US\n",
       "3  The product shall be available during normal b...    A\n",
       "4  If projected  the data must be understandable....   US"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loading in promise dataset\n",
    "df = pd.read_csv(\"datasets/promise.csv\")\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Re-Labeling</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INPUT</th>\n",
       "      <th>TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The system shall refresh the display every 60 ...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The application shall match the color of the s...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If projected  the data must be readable.  On a...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The product shall be available during normal b...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>If projected  the data must be understandable....</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The product shall ensure that it can only be a...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The product shall be intuitive and self-explan...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>The product shall respond fast to keep up-to-d...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>The system shall have a MDI form that allows f...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>The system shall display Events in a vertical ...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>The system shall display the Events in a graph...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>The system shall display Events or Activities.</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>The display shall have two regions:  left 2/3 ...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>The data displayed in both the nodes within th...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>The table side of the display shall be split i...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>The top 1/4 of the table will hold events that...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>The bottom 3/4 of the table will hold events t...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>The system shall color code events according t...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>The system shall display data from the Sync Ma...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>The system shall link Events back to either th...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                INPUT TYPE\n",
       "0   The system shall refresh the display every 60 ...   NF\n",
       "1   The application shall match the color of the s...   NF\n",
       "2   If projected  the data must be readable.  On a...   NF\n",
       "3   The product shall be available during normal b...   NF\n",
       "4   If projected  the data must be understandable....   NF\n",
       "5   The product shall ensure that it can only be a...   NF\n",
       "6   The product shall be intuitive and self-explan...   NF\n",
       "7   The product shall respond fast to keep up-to-d...   NF\n",
       "8   The system shall have a MDI form that allows f...    F\n",
       "9   The system shall display Events in a vertical ...    F\n",
       "10  The system shall display the Events in a graph...    F\n",
       "11     The system shall display Events or Activities.    F\n",
       "12  The display shall have two regions:  left 2/3 ...    F\n",
       "13  The data displayed in both the nodes within th...    F\n",
       "14  The table side of the display shall be split i...    F\n",
       "15  The top 1/4 of the table will hold events that...    F\n",
       "16  The bottom 3/4 of the table will hold events t...    F\n",
       "17  The system shall color code events according t...    F\n",
       "18  The system shall display data from the Sync Ma...    F\n",
       "19  The system shall link Events back to either th...    F"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Making dataset binary between functional & non-functional requirements\n",
    "\n",
    "df.loc[df['TYPE'] != 'F', 'TYPE'] = 'NF' # Locating all values that do not equal 'F' and changing them to 'NF'\n",
    "\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Lowercasing</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INPUT</th>\n",
       "      <th>TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the system shall refresh the display every 60 ...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the application shall match the color of the s...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>if projected  the data must be readable.  on a...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>the product shall be available during normal b...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>if projected  the data must be understandable....</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               INPUT TYPE\n",
       "0  the system shall refresh the display every 60 ...   NF\n",
       "1  the application shall match the color of the s...   NF\n",
       "2  if projected  the data must be readable.  on a...   NF\n",
       "3  the product shall be available during normal b...   NF\n",
       "4  if projected  the data must be understandable....   NF"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['INPUT'] = df['INPUT'].str.lower()\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Stopword Removal</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INPUT</th>\n",
       "      <th>TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>system shall refresh display every 60 seconds.</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>application shall match color schema set forth...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>projected data must readable. 10x10 projection...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>product shall available normal business hours....</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>projected data must understandable. 10x10 proj...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               INPUT TYPE\n",
       "0     system shall refresh display every 60 seconds.   NF\n",
       "1  application shall match color schema set forth...   NF\n",
       "2  projected data must readable. 10x10 projection...   NF\n",
       "3  product shall available normal business hours....   NF\n",
       "4  projected data must understandable. 10x10 proj...   NF"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "#nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "# Define stop words\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "# Function to remove stopwords\n",
    "def remove_stopwords(text):\n",
    "    words = text.split()  # Split the text into words\n",
    "    filtered_text = ' '.join([word for word in words if word.lower() not in stop_words])\n",
    "    return filtered_text\n",
    "\n",
    "# Apply stopword removal to the DataFrame column\n",
    "df['INPUT'] = df['INPUT'].apply(remove_stopwords)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Punctuation Removal<h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INPUT</th>\n",
       "      <th>TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>system shall refresh display every 60 seconds</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>application shall match color schema set forth...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>projected data must readable 10x10 projection ...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>product shall available normal business hours ...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>projected data must understandable 10x10 proje...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>product shall ensure accessed authorized users...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>product shall intuitive selfexplanatory  90 pe...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>product shall respond fast keep uptodate data ...</td>\n",
       "      <td>NF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>system shall mdi form allows viewing graph dat...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>system shall display events vertical table time</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>system shall display events graph time</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>system shall display events activities</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>display shall two regions left 23 display grap...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>data displayed nodes within graph rows table m...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>table side display shall split 2 regions seque...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>top 14 table hold events occur sequentially</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>bottom 34 table hold events occur according re...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>system shall color code events according varia...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>system shall display data sync matrix 10 exerc...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>system shall link events back either sync matr...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                INPUT TYPE\n",
       "0       system shall refresh display every 60 seconds   NF\n",
       "1   application shall match color schema set forth...   NF\n",
       "2   projected data must readable 10x10 projection ...   NF\n",
       "3   product shall available normal business hours ...   NF\n",
       "4   projected data must understandable 10x10 proje...   NF\n",
       "5   product shall ensure accessed authorized users...   NF\n",
       "6   product shall intuitive selfexplanatory  90 pe...   NF\n",
       "7   product shall respond fast keep uptodate data ...   NF\n",
       "8   system shall mdi form allows viewing graph dat...    F\n",
       "9     system shall display events vertical table time    F\n",
       "10             system shall display events graph time    F\n",
       "11             system shall display events activities    F\n",
       "12  display shall two regions left 23 display grap...    F\n",
       "13  data displayed nodes within graph rows table m...    F\n",
       "14  table side display shall split 2 regions seque...    F\n",
       "15        top 14 table hold events occur sequentially    F\n",
       "16  bottom 34 table hold events occur according re...    F\n",
       "17  system shall color code events according varia...    F\n",
       "18  system shall display data sync matrix 10 exerc...    F\n",
       "19  system shall link events back either sync matr...    F"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "# Replacing percent symbol to string\n",
    "df['INPUT'] = df['INPUT'].str.replace('%', ' percent')\n",
    "\n",
    "# Remove punctuation from a specific column (e.g., 'Column1')\n",
    "df['INPUT'] = df['INPUT'].apply(lambda x: x.translate(str.maketrans('', '', string.punctuation)) if isinstance(x, str) else x)\n",
    "\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Tokenize Dataset</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[    0, 11873,  1199,  ...,     1,     1,     1],\n",
      "        [    0, 31867,  1199,  ...,     1,     1,     1],\n",
      "        [    0,  3887,  2331,  ...,     1,     1,     1],\n",
      "        ...,\n",
      "        [    0,   821,    86,  ...,     1,     1,     1],\n",
      "        [    0, 16171,   754,  ...,     1,     1,     1],\n",
      "        [    0, 17655,   382,  ...,     1,     1,     1]]), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0]])}\n"
     ]
    }
   ],
   "source": [
    "# Tokenize the dataset with the appropriate settings\n",
    "tokenized_dataset = tokenizer(\n",
    "    df['INPUT'].tolist(),\n",
    "    padding=True,           # Add padding to make the input sequences the same length\n",
    "    truncation=True,        # Truncate sequences to the maximum length allowed by the model\n",
    "    max_length=512,         # Adjust max_length based on model's input size\n",
    "    return_tensors=\"pt\"     # Return PyTorch tensors\n",
    ")\n",
    "\n",
    "# Print tokenized output (optional)\n",
    "print(tokenized_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Model Fitting<h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Label Descriptions<h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label descriptions for Functional and Non-Functional\n",
    "\n",
    "label_descriptions = {\n",
    "    \"Functional\": [\n",
    "        \"Specifies the tasks or actions the system must perform to meet user objectives, such as processing data or responding to inputs.\", \n",
    "        \"Defines interactions between users and the system, focusing on activities like data input, processing, and system responses.\",\n",
    "        \"Describes functions the system must execute to provide its core features, enabling users to complete their goals.\",\n",
    "        \"Outlines precise operations the system must carry out, detailing steps to complete processes or respond to user commands.\",\n",
    "        \"Lists functions that directly support user needs, such as generating reports, calculating results, or saving information.\",\n",
    "        \"States requirements for actions the system must perform, including data validation, transaction handling, and workflow management.\",\n",
    "        \"Defines specific behaviors or responses expected from the system based on various user inputs and interactions.\"\n",
    "        ],\n",
    "    \"Non-Functional\": [\n",
    "        \"Specifies criteria for system performance, including processing speed, load handling, and response time under different conditions.\", \n",
    "        \"Defines security standards, such as data encryption, access control, and user authentication to ensure data protection.\",\n",
    "        \"Describes usability requirements, including user interface consistency, accessibility, and ease of navigation for end-users.\",\n",
    "        \"Outlines reliability standards, such as system uptime, fault tolerance, and recovery mechanisms to ensure consistent operation.\",\n",
    "        \"Specifies scalability requirements, including the system's ability to handle increased data volume or users without performance loss.\",\n",
    "        \"Sets efficiency benchmarks, such as memory usage, storage requirements, and energy consumption limits for optimal resource use.\",\n",
    "        \"Establishes compliance standards, specifying adherence to industry regulations or corporate guidelines to meet legal requirements.\"\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Label Embeddings</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate embeddings for each description in both Functional and Non-Functional categories\n",
    "label_embeddings = {\n",
    "    \"Functional\": [sbert.encode(desc) for desc in label_descriptions[\"Functional\"]],\n",
    "    \"Non-Functional\": [sbert.encode(desc) for desc in label_descriptions[\"Non-Functional\"]]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Preparing Data Loader & Tensor Dataset</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "label_mapping = {\"F\": 0, \"NF\": 1}\n",
    "numerical_labels = df['TYPE'].map(label_mapping).tolist()\n",
    "\n",
    "# Prepare a TensorDataset with input IDs and attention masks\n",
    "input_ids = tokenized_dataset[\"input_ids\"]\n",
    "attention_mask = tokenized_dataset[\"attention_mask\"]\n",
    "labels = torch.tensor(numerical_labels)\n",
    "dataset_torch = TensorDataset(input_ids, attention_mask, labels)\n",
    "dataloader = DataLoader(dataset_torch, batch_size=8, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>CLS Pooling & Cosine Similarity</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify_requirements(hidden_states, label_embeddings):\n",
    "\n",
    "    # Apply CLS pooling by taking the first token's embedding (CLS token)\n",
    "    cls_embeddings = hidden_states[:, 0, :]  # Shape: (batch_size, hidden_size)\n",
    "\n",
    "    # Classify each requirement in the batch\n",
    "    classifications = []\n",
    "    for cls_embedding in cls_embeddings:\n",
    "        # Convert tensor to numpy for compatibility with cosine similarity calculation\n",
    "        cls_embedding_np = cls_embedding.cpu().numpy()\n",
    "\n",
    "        # Calculate similarity for functional and non-functional descriptions\n",
    "        functional_similarities = [util.cos_sim(cls_embedding_np, emb)[0][0].item() for emb in label_embeddings[\"Functional\"]]\n",
    "        non_functional_similarities = [util.cos_sim(cls_embedding_np, emb)[0][0].item() for emb in label_embeddings[\"Non-Functional\"]]\n",
    "\n",
    "        # Calculate maximum similarity for each label\n",
    "        max_functional_similarity = max(functional_similarities)\n",
    "        max_non_functional_similarity = max(non_functional_similarities)\n",
    "\n",
    "        # Determine classification based on higher maximum similarity\n",
    "        if max_functional_similarity > max_non_functional_similarity:\n",
    "            classifications.append(\"F\")\n",
    "        else:\n",
    "            classifications.append(\"NF\")\n",
    "\n",
    "    return classifications"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Evaluation & Testing</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 54.88%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.45      0.53      0.49       255\n",
      "           1       0.63      0.56      0.60       370\n",
      "\n",
      "    accuracy                           0.55       625\n",
      "   macro avg       0.54      0.55      0.54       625\n",
      "weighted avg       0.56      0.55      0.55       625\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "prediction_mapping = {\"F\": 0, \"NF\": 1}\n",
    "\n",
    "true_labels = []  # This will collect true labels\n",
    "\n",
    "# Iterate over the DataLoader to get embeddings and classify\n",
    "all_predictions = []\n",
    "for batch in dataloader:\n",
    "    input_ids, attention_mask, labels = batch  # Ensure labels are included in the batch\n",
    "    input_ids, attention_mask = input_ids.to(device), attention_mask.to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = bert4re(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        hidden_states = outputs.last_hidden_state\n",
    "\n",
    "    # Classify requirements in the batch\n",
    "    predictions = classify_requirements(hidden_states, label_embeddings)\n",
    "    all_predictions.extend(predictions)\n",
    "    true_labels.extend(labels.tolist())  # Append true labels from the batch\n",
    "\n",
    "# Convert string predictions to numerical values\n",
    "numerical_predictions = [prediction_mapping[pred] for pred in all_predictions]\n",
    "\n",
    "# Ensure the true_labels list is populated correctly\n",
    "if len(true_labels) != len(numerical_predictions):\n",
    "    print(f\"Error: Mismatch in true_labels ({len(true_labels)}) and predictions ({len(numerical_predictions)})\")\n",
    "else:\n",
    "    # Evaluate accuracy\n",
    "    accuracy = accuracy_score(true_labels, numerical_predictions)\n",
    "    print(f\"Accuracy: {accuracy * 100:.2f}%\")\n",
    "\n",
    "    # Generate classification report\n",
    "    print(classification_report(true_labels, numerical_predictions))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}