step2_initial_ranking.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "06e3f592-0883-4eed-be7d-99250dee7202",
   "metadata": {},
   "source": [
    "# Step 2: BM25 top 20 evidences selection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6187207-ba5c-464e-828e-3bb328622568",
   "metadata": {},
   "source": [
    "# Readme\n",
    "*This notebook focusing on the initial top 20 evidences ranking by BM25, and output the data that contains the top 20 evidence for each claim*"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "358322a7-4970-443c-b33f-1c2c50f89b47",
   "metadata": {},
   "source": [
    "# 1.DataSet Processing\n",
    "(You can add as many code blocks and text blocks as you need. However, YOU SHOULD NOT MODIFY the section title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3691cfd4-804c-42eb-99f0-f2c6783b8277",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from collections import Counter\n",
    "import ast\n",
    "import spacy\n",
    "import string\n",
    "import time\n",
    "from rank_bm25 import BM25Okapi\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "429013ff-ee98-441c-9213-ec1f2386e22a",
   "metadata": {},
   "source": [
    "Please modify the data path that produced from the step 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "66c17e7f-354e-445e-be9e-fbf0dfe1083c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data_folder/train.csv\")\n",
    "evidence = pd.read_csv(\"data_folder/evidence.csv\")\n",
    "dev = pd.read_csv(\"data_folder/dev.csv\")\n",
    "test = pd.read_csv(\"data_folder/test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b9b089f-1eef-40f7-a4f7-91928ab30504",
   "metadata": {},
   "source": [
    "# Stop word and punct removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b218bd52-ad88-4854-ba07-e45d3f344b52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the SpaCy model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "# Prepare a set of punctuation marks for removal\n",
    "punctuations = string.punctuation\n",
    "\n",
    "def spacy_lemmatize_text(input_text):\n",
    "    # Remove punctuation and convert to lowercase\n",
    "    translator = str.maketrans('', '', string.punctuation)\n",
    "    input_text = input_text.lower().translate(translator)\n",
    "    \n",
    "    # Process text using SpaCy\n",
    "    doc = nlp(input_text)\n",
    "    \n",
    "    # Lemmatize text and remove stopwords, punctuation, specified POS tags\n",
    "    lemmatized_tokens = [token.lemma_ for token in doc if not token.is_stop]\n",
    "    \n",
    "    # Join words back into one string\n",
    "    return ' '.join(lemmatized_tokens)\n",
    "\n",
    "# Example usage for preprocessing and timing\n",
    "import time\n",
    "\n",
    "def process_and_time(df, column_name):\n",
    "    start_time = time.time()\n",
    "    df[column_name] = df[column_name].apply(spacy_lemmatize_text)\n",
    "    print(f\"{column_name.capitalize()} lemmatize finished in {time.time() - start_time:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6f9eaa77-da59-4821-88e2-5c769803a35b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Claim_text lemmatize finished in 0.70 seconds\n",
      "Claim_text lemmatize finished in 0.54 seconds\n",
      "Claim_text lemmatize finished in 3.94 seconds\n",
      "Evidence_text lemmatize finished in 3548.07 seconds\n"
     ]
    }
   ],
   "source": [
    "# Applying to each DataFrame\n",
    "process_and_time(dev, 'claim_text')\n",
    "process_and_time(test, 'claim_text')\n",
    "process_and_time(train, 'claim_text')\n",
    "process_and_time(evidence, 'evidence_text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dfbee4ed-4cc6-4a08-97d4-3ba5a7778c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rearrange the evidences into a list of integer\n",
    "train['evidences'] = train['evidences'].apply(lambda x: [int(e.split('-')[-1]) for e in eval(x)])\n",
    "dev['evidences'] = dev['evidences'].apply(lambda x: [int(e.split('-')[-1]) for e in eval(x)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53c61639-1227-446e-9443-3ae15c924522",
   "metadata": {},
   "source": [
    "# 2. Model Implementation\n",
    "(You can add as many code blocks and text blocks as you need. However, YOU SHOULD NOT MODIFY the section title)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eead4e12-421c-4fb3-b90e-a632bdc767f3",
   "metadata": {},
   "source": [
    "# Search the most similar 20 evidences for each claim by BM25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c923aacc-4574-493a-82db-e83be8b350c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_evidence(evidence_df):\n",
    "    start_time = time.time()\n",
    "    evidence_df['evidence_text'] = evidence_df['evidence_text'].astype(str)\n",
    "    tokenized_evidence = [word_tokenize(doc.lower()) if doc != 'nan' else [] for doc in evidence_df['evidence_text']]\n",
    "    print(f\"Tokenized evidence finished in {time.time() - start_time:.2f} seconds\")\n",
    "    return tokenized_evidence\n",
    "\n",
    "def prefilter_top_evidence_with_bm25(df, tokenized_evidence, top_k=20, k1=1.2, b=0.75):\n",
    "    start_time = time.time()\n",
    "    # Initialize BM25 with dynamic k1 and b values\n",
    "    bm25 = BM25Okapi(tokenized_evidence, k1=k1, b=b)\n",
    "    \n",
    "    # Tokenize the claim texts and query the BM25 model\n",
    "    df['prefilter_evidence'] = df['claim_text'].apply(lambda x: query_bm25(x, bm25, top_k))\n",
    "    print(f\"Retrieved top {top_k} evidence finished in {time.time() - start_time:.2f} seconds\")\n",
    "    return df\n",
    "\n",
    "def query_bm25(query, bm25, top_k):\n",
    "    # Tokenize the query\n",
    "    tokenized_query = word_tokenize(query.lower())\n",
    "    \n",
    "    # Get scores and sort by scores\n",
    "    doc_scores = bm25.get_scores(tokenized_query)\n",
    "    return sorted(range(len(doc_scores)), key=lambda i: doc_scores[i], reverse=True)[:top_k]\n",
    "\n",
    "# The evaluation function is directly retrieved from the eval.py \n",
    "def print_retrieval_result(df, top_k):\n",
    "    evidence_recall_scores = []\n",
    "    evidence_precision_scores = []\n",
    "    evidence_fscore_scores = []\n",
    "    \n",
    "    # Evaluate evidence retrieval\n",
    "    for index, row in df.iterrows():\n",
    "        true_evidences = set(row['evidences'])\n",
    "        predicted_evidences = set(row['prefilter_evidence'][:top_k])\n",
    "        \n",
    "        evidence_correct = len(true_evidences & predicted_evidences)\n",
    "        \n",
    "        # Calculate recall, precision, and F-score\n",
    "        evidence_recall = evidence_correct / len(true_evidences) if true_evidences else 0\n",
    "        evidence_precision = evidence_correct / len(predicted_evidences) if predicted_evidences else 0\n",
    "        evidence_fscore = (2 * evidence_precision * evidence_recall) / (evidence_precision + evidence_recall) if (evidence_recall + evidence_precision) > 0 else 0\n",
    "        \n",
    "        # Store the scores\n",
    "        evidence_recall_scores.append(evidence_recall)\n",
    "        evidence_precision_scores.append(evidence_precision)\n",
    "        evidence_fscore_scores.append(evidence_fscore)\n",
    "    \n",
    "    # Print mean scores\n",
    "    print(f\"Mean Recall: {np.mean(evidence_recall_scores)}|, Mean Precision: {np.mean(evidence_precision_scores)}, Mean F-Score: {np.mean(evidence_fscore_scores)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "143454ff-b7fb-4936-9e5d-ed0f306151f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenized evidence finished in 56.30 seconds\n"
     ]
    }
   ],
   "source": [
    "tokenized_evidence_lemmatize = tokenize_evidence(evidence)  # Tokenize once outside the loop"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6399fda-537e-4465-968d-72939db0f1cc",
   "metadata": {},
   "source": [
    "# 3.Testing and Evaluation\n",
    "(You can add as many code blocks and text blocks as you need. However, YOU SHOULD NOT MODIFY the section title)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee69cd56-c827-4299-9d41-526f8cc53505",
   "metadata": {},
   "source": [
    "Testing the default parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5ab5ad3a-8ba6-4856-9966-6dd7e811f952",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing k=1.2, b=0.75\n",
      "Retrieved top 20 evidence finished in 2407.07 seconds\n",
      "Retrieved top 20 evidence finished in 328.27 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2607084690553746|, Mean Precision: 0.041164495114006515, Mean F-Score: 0.06922177088831954\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.11400651465798045|, Mean Precision: 0.11726384364820845, Mean F-Score: 0.10818985574685898\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.32283549783549786|, Mean Precision: 0.04642857142857143, Mean F-Score: 0.07873704781948714\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14004329004329003|, Mean Precision: 0.12987012987012986, Mean F-Score: 0.12422696351267778\n"
     ]
    }
   ],
   "source": [
    "k = 1.2\n",
    "b = 0.75\n",
    "print(f\"Testing k={k}, b={b}\")\n",
    "train_df = prefilter_top_evidence_with_bm25(train, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "dev_df = prefilter_top_evidence_with_bm25(dev, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "\n",
    "print(f\"Result of top 20 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=20)  # For top 20\n",
    "print(f\"Result of top 3 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=3)   # For top 3\n",
    "\n",
    "print(f\"Result of top 20 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=20)  # For top 20\n",
    "print(f\"Result of top 3 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=3)   # For top 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49d62c0b-c241-4f45-8ea5-218719d6409e",
   "metadata": {},
   "source": [
    "# hyperparameter search\n",
    "*In BM25, higher K1 means higher weighted on term frequency, higher b means higher weighted on sentence length*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0deec54b-bf88-431e-ad2b-00acc88532ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing k=0.6, b=0.75\n",
      "Retrieved top 20 evidence finished in 1989.80 seconds\n",
      "Retrieved top 20 evidence finished in 264.71 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28498914223669924, Mean Precision: 0.04466612377850163, Mean F-Score: 0.07514823820375442\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12935667752442995, Mean Precision: 0.13083604777415853, Mean F-Score: 0.12135877152163796\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3584415584415584, Mean Precision: 0.05097402597402597, Mean F-Score: 0.08648700443166847\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1448051948051948, Mean Precision: 0.13419913419913418, Mean F-Score: 0.1283395176252319\n",
      "Testing k=0.6, b=0.825\n",
      "Retrieved top 20 evidence finished in 1989.44 seconds\n",
      "Retrieved top 20 evidence finished in 263.99 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2824104234527687, Mean Precision: 0.04429967426710098, Mean F-Score: 0.07452929664868485\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.1285830618892508, Mean Precision: 0.13002171552660152, Mean F-Score: 0.12059678920428107\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3517316017316018, Mean Precision: 0.05000000000000001, Mean F-Score: 0.08481255240148522\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14664502164502163, Mean Precision: 0.1385281385281385, Mean F-Score: 0.13138528138528138\n",
      "Testing k=0.6, b=0.9\n",
      "Retrieved top 20 evidence finished in 1987.62 seconds\n",
      "Retrieved top 20 evidence finished in 264.54 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2743892508143323, Mean Precision: 0.043118892508143325, Mean F-Score: 0.07253480799048001\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12661509229098805, Mean Precision: 0.1278501628664495, Mean F-Score: 0.11872188614859626\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.35205627705627707, Mean Precision: 0.049999999999999996, Mean F-Score: 0.08483419742313024\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1475108225108225, Mean Precision: 0.1385281385281385, Mean F-Score: 0.13192640692640695\n",
      "Testing k=0.7, b=0.6\n",
      "Retrieved top 20 evidence finished in 1989.11 seconds\n",
      "Retrieved top 20 evidence finished in 264.83 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28612920738327907, Mean Precision: 0.04499185667752443, Mean F-Score: 0.07567745147266462\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.1267915309446254, Mean Precision: 0.12866449511400652, Mean F-Score: 0.1191949744067008\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.36320346320346325, Mean Precision: 0.05259740259740261, Mean F-Score: 0.08899406259090055\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1396103896103896, Mean Precision: 0.132034632034632, Mean F-Score: 0.12476808905380335\n",
      "Testing k=0.7, b=0.675\n",
      "Retrieved top 20 evidence finished in 1984.36 seconds\n",
      "Retrieved top 20 evidence finished in 265.09 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28478555917481, Mean Precision: 0.04462540716612378, Mean F-Score: 0.07508037718312469\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12605863192182412, Mean Precision: 0.12812160694896849, Mean F-Score: 0.1185784085621219\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3604978354978355, Mean Precision: 0.051623376623376634, Mean F-Score: 0.08750243827318928\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14155844155844155, Mean Precision: 0.132034632034632, Mean F-Score: 0.1257421150278293\n",
      "Testing k=0.7, b=0.75\n",
      "Retrieved top 20 evidence finished in 1988.82 seconds\n",
      "Retrieved top 20 evidence finished in 264.34 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2810260586319218, Mean Precision: 0.04413680781758958, Mean F-Score: 0.0742508725754725\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12829804560260585, Mean Precision: 0.12920738327904452, Mean F-Score: 0.12007910656119125\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.35519480519480523, Mean Precision: 0.05064935064935065, Mean F-Score: 0.08589668565953153\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14534632034632033, Mean Precision: 0.13636363636363635, Mean F-Score: 0.12976190476190477\n",
      "Testing k=0.7, b=0.825\n",
      "Retrieved top 20 evidence finished in 1987.38 seconds\n",
      "Retrieved top 20 evidence finished in 264.63 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2743213897937025, Mean Precision: 0.04311889250814332, Mean F-Score: 0.07253453976510202\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12409066232356133, Mean Precision: 0.1259500542888165, Mean F-Score: 0.1168082053668373\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.34913419913419913, Mean Precision: 0.049350649350649346, Mean F-Score: 0.08377359136252419\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1475108225108225, Mean Precision: 0.1385281385281385, Mean F-Score: 0.13192640692640695\n",
      "Testing k=0.7, b=0.9\n",
      "Retrieved top 20 evidence finished in 1990.23 seconds\n",
      "Retrieved top 20 evidence finished in 264.81 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2716883821932682, Mean Precision: 0.04279315960912052, Mean F-Score: 0.07196941494987098\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12289630836047774, Mean Precision: 0.1245928338762215, Mean F-Score: 0.1155382348379091\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3404761904761905, Mean Precision: 0.048701298701298704, Mean F-Score: 0.08259050943410062\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1475108225108225, Mean Precision: 0.1385281385281385, Mean F-Score: 0.13192640692640695\n",
      "Testing k=0.8, b=0.6\n",
      "Retrieved top 20 evidence finished in 1989.92 seconds\n",
      "Retrieved top 20 evidence finished in 265.07 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28223398479913137, Mean Precision: 0.04434039087947883, Mean F-Score: 0.07459655071360213\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.1264115092290988, Mean Precision: 0.12866449511400652, Mean F-Score: 0.11899526911741896\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3593073593073593, Mean Precision: 0.05162337662337663, Mean F-Score: 0.08743562103245898\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14025974025974028, Mean Precision: 0.12987012987012986, Mean F-Score: 0.12411873840445269\n",
      "Testing k=0.8, b=0.675\n",
      "Retrieved top 20 evidence finished in 1989.61 seconds\n",
      "Retrieved top 20 evidence finished in 264.92 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.281514657980456, Mean Precision: 0.04421824104234527, Mean F-Score: 0.07437580735838772\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12570575461454941, Mean Precision: 0.12703583061889248, Mean F-Score: 0.11790561501473555\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.35378787878787876, Mean Precision: 0.05032467532467533, Mean F-Score: 0.08533605393289188\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14047619047619048, Mean Precision: 0.132034632034632, Mean F-Score: 0.12530921459492889\n",
      "Testing k=0.8, b=0.75\n",
      "Retrieved top 20 evidence finished in 1994.80 seconds\n",
      "Retrieved top 20 evidence finished in 264.58 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2740770901194354, Mean Precision: 0.043118892508143325, Mean F-Score: 0.07252565614058322\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12357491856677524, Mean Precision: 0.1262214983713355, Mean F-Score: 0.11682177757096324\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.35097402597402594, Mean Precision: 0.04967532467532467, Mean F-Score: 0.08431659907944494\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1462121212121212, Mean Precision: 0.13636363636363635, Mean F-Score: 0.13030303030303034\n",
      "Testing k=0.8, b=0.825\n",
      "Retrieved top 20 evidence finished in 1985.59 seconds\n",
      "Retrieved top 20 evidence finished in 264.55 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2698154180238871, Mean Precision: 0.042467426710097726, Mean F-Score: 0.07142504618074651\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12128121606948969, Mean Precision: 0.12350705754614547, Mean F-Score: 0.11435939196525516\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.341017316017316, Mean Precision: 0.048701298701298704, Mean F-Score: 0.08261403663154088\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1462121212121212, Mean Precision: 0.13636363636363635, Mean F-Score: 0.13030303030303034\n",
      "Testing k=0.8, b=0.9\n",
      "Retrieved top 20 evidence finished in 1987.82 seconds\n",
      "Retrieved top 20 evidence finished in 264.98 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2655130293159609, Mean Precision: 0.041938110749185666, Mean F-Score: 0.07051561041045533\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.11735884907709013, Mean Precision: 0.12024972855591748, Mean F-Score: 0.11100124088723436\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3268398268398268, Mean Precision: 0.04707792207792208, Mean F-Score: 0.07982331991366434\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14837662337662336, Mean Precision: 0.13852813852813853, Mean F-Score: 0.13246753246753246\n",
      "Testing k=0.9, b=0.6\n",
      "Retrieved top 20 evidence finished in 1988.15 seconds\n",
      "Retrieved top 20 evidence finished in 265.38 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28045602605863196, Mean Precision: 0.04421824104234527, Mean F-Score: 0.07436255549199892\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12562432138979368, Mean Precision: 0.12676438653637348, Mean F-Score: 0.11757988211571271\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.35638528138528136, Mean Precision: 0.05097402597402597, Mean F-Score: 0.08637501497185292\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14242424242424243, Mean Precision: 0.132034632034632, Mean F-Score: 0.12628324056895487\n",
      "Testing k=0.9, b=0.675\n",
      "Retrieved top 20 evidence finished in 1991.40 seconds\n",
      "Retrieved top 20 evidence finished in 264.68 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2743621064060804, Mean Precision: 0.043322475570032576, Mean F-Score: 0.0728498057435177\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.12356134636264929, Mean Precision: 0.12567861020629748, Mean F-Score: 0.11629052272374747\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.35378787878787876, Mean Precision: 0.05032467532467533, Mean F-Score: 0.08533605393289188\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14134199134199132, Mean Precision: 0.13203463203463203, Mean F-Score: 0.12585034013605442\n",
      "Testing k=0.9, b=0.75\n",
      "Retrieved top 20 evidence finished in 1991.77 seconds\n",
      "Retrieved top 20 evidence finished in 264.52 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2708469055374593, Mean Precision: 0.04267100977198697, Mean F-Score: 0.07176210891973532\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.1210097719869707, Mean Precision: 0.12350705754614548, Mean F-Score: 0.11425081433224754\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3475108225108225, Mean Precision: 0.04902597402597402, Mean F-Score: 0.08323246582139865\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1462121212121212, Mean Precision: 0.13636363636363635, Mean F-Score: 0.13030303030303034\n",
      "Testing k=0.9, b=0.825\n",
      "Retrieved top 20 evidence finished in 1992.11 seconds\n",
      "Retrieved top 20 evidence finished in 265.53 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.26579804560260584, Mean Precision: 0.04197882736156352, Mean F-Score: 0.070570001918961\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.11828175895765472, Mean Precision: 0.12106406080347448, Mean F-Score: 0.11186016751977663\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3243506493506494, Mean Precision: 0.046753246753246755, Mean F-Score: 0.07923702215345355\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.1462121212121212, Mean Precision: 0.13636363636363635, Mean F-Score: 0.1303030303030303\n",
      "Testing k=0.9, b=0.9\n",
      "Retrieved top 20 evidence finished in 1991.22 seconds\n",
      "Retrieved top 20 evidence finished in 265.00 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.26359934853420197, Mean Precision: 0.041530944625407164, Mean F-Score: 0.06985254654706574\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.11282573289902278, Mean Precision: 0.11563517915309446, Mean F-Score: 0.10669497440670077\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.32359307359307354, Mean Precision: 0.046753246753246755, Mean F-Score: 0.0792330011415274\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14805194805194805, Mean Precision: 0.13852813852813853, Mean F-Score: 0.1322356215213358\n"
     ]
    }
   ],
   "source": [
    "k_values = [0.6, 0.7, 0.8, 0.9]\n",
    "b_values = [0.6, 0.675, 0.75, 0.825, 0.9]\n",
    "\n",
    "for k in k_values:\n",
    "    for b in b_values:\n",
    "        if k == 0.6:\n",
    "            if b ==0.6 or b == 0.675:\n",
    "                continue\n",
    "        print(f\"Testing k={k}, b={b}\")\n",
    "        train_df = prefilter_top_evidence_with_bm25(train, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "        dev_df = prefilter_top_evidence_with_bm25(dev, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "        \n",
    "        print(f\"Result of top 20 performance of train: \")\n",
    "        print_retrieval_result(train_df, top_k=20)  # For top 20\n",
    "        print(f\"Result of top 3 performance of train: \")\n",
    "        print_retrieval_result(train_df, top_k=3)   # For top 3\n",
    "\n",
    "        print(f\"Result of top 20 performance of dev: \")\n",
    "        print_retrieval_result(dev_df, top_k=20)  # For top 20\n",
    "        print(f\"Result of top 3 performance of dev: \")\n",
    "        print_retrieval_result(dev_df, top_k=3)   # For top 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de4975e4-1523-495f-85ca-e5d526656c09",
   "metadata": {},
   "source": [
    "*By Oberseved result, we suspect k1 = 0.5, and b = 0.85 could compromise the result between train and dev*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f8cd0164-2deb-4a82-bf69-d8cd5ae755b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing k=0.5, b=0.85\n",
      "Retrieved top 20 evidence finished in 2473.02 seconds\n",
      "Retrieved top 20 evidence finished in 347.93 seconds\n",
      "Retrieved top 20 evidence finished in 320.24 seconds\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28584419109663406, Mean Precision: 0.044828990228013024, Mean F-Score: 0.07541499370666628\n",
      "Result of top 3 performance of train: \n",
      "Mean Recall: 0.1294516829533116, Mean Precision: 0.1305646036916395, Mean F-Score: 0.12113967736931906\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3617965367965368, Mean Precision: 0.05194805194805195, Mean F-Score: 0.08802191879266978\n",
      "Result of top 3 performance of dev: \n",
      "Mean Recall: 0.14642857142857144, Mean Precision: 0.13636363636363635, Mean F-Score: 0.13019480519480517\n"
     ]
    }
   ],
   "source": [
    "k = 0.5\n",
    "b = 0.85\n",
    "print(f\"Testing k={k}, b={b}\")\n",
    "train_df = prefilter_top_evidence_with_bm25(train, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "dev_df = prefilter_top_evidence_with_bm25(dev, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "test_df = prefilter_top_evidence_with_bm25(test, tokenized_evidence_lemmatize, top_k=20, k1=k, b=b)\n",
    "print(f\"Result of top 20 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=20)  # For top 20\n",
    "print(f\"Result of top 3 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=3)   # For top 3\n",
    "\n",
    "print(f\"Result of top 20 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=20)  # For top 20\n",
    "print(f\"Result of top 3 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=3)   # For top 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03c991e0-bbae-471a-9fc7-9b1f55a256fd",
   "metadata": {},
   "source": [
    "The reason we only keep top 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b4b3c33e-d0e2-4d03-9f6a-69c7819424b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing k=1.2, b=0.75\n",
      "Retrieved top 100 evidence finished in 2164.95 seconds\n",
      "Retrieved top 100 evidence finished in 277.05 seconds\n",
      "Retrieved top 100 evidence finished in 247.59 seconds\n",
      "Result of top 100 performance of train: \n",
      "Mean Recall: 0.42498642779587414|, Mean Precision: 0.013534201954397392, Mean F-Score: 0.026056002395988546\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.2607084690553746|, Mean Precision: 0.041164495114006515, Mean F-Score: 0.06922177088831954\n",
      "Result of top 100 performance of dev: \n",
      "Mean Recall: 0.5059523809523809|, Mean Precision: 0.01538961038961039, Mean F-Score: 0.0296558379164439\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.32283549783549786|, Mean Precision: 0.04642857142857143, Mean F-Score: 0.07873704781948714\n",
      "Testing k=0.5, b=0.85\n",
      "Retrieved top 100 evidence finished in 2027.50 seconds\n",
      "Retrieved top 100 evidence finished in 278.58 seconds\n",
      "Retrieved top 100 evidence finished in 260.99 seconds\n",
      "Result of top 100 performance of train: \n",
      "Mean Recall: 0.4506650380021716|, Mean Precision: 0.01444625407166124, Mean F-Score: 0.02780881984481913\n",
      "Result of top 20 performance of train: \n",
      "Mean Recall: 0.28584419109663406|, Mean Precision: 0.044828990228013024, Mean F-Score: 0.07541499370666628\n",
      "Result of top 100 performance of dev: \n",
      "Mean Recall: 0.5164502164502165|, Mean Precision: 0.01590909090909091, Mean F-Score: 0.030646490814170303\n",
      "Result of top 20 performance of dev: \n",
      "Mean Recall: 0.3617965367965368|, Mean Precision: 0.05194805194805195, Mean F-Score: 0.08802191879266978\n"
     ]
    }
   ],
   "source": [
    "k = 1.2\n",
    "b = 0.75\n",
    "print(f\"Testing k={k}, b={b}\")\n",
    "train_df = prefilter_top_evidence_with_bm25(train, tokenized_evidence_lemmatize, top_k=100, k1=k, b=b)\n",
    "dev_df = prefilter_top_evidence_with_bm25(dev, tokenized_evidence_lemmatize, top_k=100, k1=k, b=b)\n",
    "test_df = prefilter_top_evidence_with_bm25(test, tokenized_evidence_lemmatize, top_k=100, k1=k, b=b)\n",
    "print(f\"Result of top 100 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=100)  # For top 100\n",
    "print(f\"Result of top 20 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=20)   # For top 20\n",
    "\n",
    "print(f\"Result of top 100 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=100)  # For top 100\n",
    "print(f\"Result of top 20 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=20)   # For top 20\n",
    "\n",
    "k = 0.5\n",
    "b = 0.85\n",
    "print(f\"Testing k={k}, b={b}\")\n",
    "train_df = prefilter_top_evidence_with_bm25(train, tokenized_evidence_lemmatize, top_k=100, k1=k, b=b)\n",
    "dev_df = prefilter_top_evidence_with_bm25(dev, tokenized_evidence_lemmatize, top_k=100, k1=k, b=b)\n",
    "test_df = prefilter_top_evidence_with_bm25(test, tokenized_evidence_lemmatize, top_k=100, k1=k, b=b)\n",
    "print(f\"Result of top 100 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=100)  # For top 100\n",
    "print(f\"Result of top 20 performance of train: \")\n",
    "print_retrieval_result(train_df, top_k=20)   # For top 20\n",
    "\n",
    "print(f\"Result of top 100 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=100)  # For top 100\n",
    "print(f\"Result of top 20 performance of dev: \")\n",
    "print_retrieval_result(dev_df, top_k=20)   # For top 20"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfcf1e02-1c2a-4641-a625-781aa898a9a3",
   "metadata": {},
   "source": [
    "we could found that top 20 contains most of the true evidence that BM25 could found, if we want rerank base on more evidence, then it will introduce\n",
    "even higher imbalanced(bring too much false evidence to model) to the training data."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2360162e-8713-4274-badf-c1e4381ceec5",
   "metadata": {},
   "source": [
    "# Step 3 notebook will use the dataframe that contains the top 20 evidences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1d382839-d583-4651-812d-2a3363c2dcb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv(\"train_k05b085_bm25_top20.csv\")\n",
    "dev_df.to_csv(\"dev_k05b085_bm25_top20.csv\")\n",
    "test_df.to_csv(\"test_k05b085_bm25_top20.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}