BLEU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab Rotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re as re\n",
    "import numpy as np\n",
    "import requests\n",
    "from urllib.parse import urlparse\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO \n",
    "github_token = \"ADD_YOUR_GITHUB_TOKEN\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Collection\n",
    "@Misc{acl-ocl,\n",
    "    author =       {Shaurya Rohatgi, Yanxia Qin, Benjamin Aw, Niranjana Unnithan, Min-Yen Kan},\n",
    "    title =        {The ACL OCL Corpus: advancing Open science in Computational Linguistics},\n",
    "    howpublished = {arXiv},\n",
    "    year =         {2022},\n",
    "    url =          {https://huggingface.co/datasets/ACL-OCL/ACL-OCL-Corpus}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acl_id</th>\n",
       "      <th>abstract</th>\n",
       "      <th>full_text</th>\n",
       "      <th>corpus_paper_id</th>\n",
       "      <th>pdf_hash</th>\n",
       "      <th>numcitedby</th>\n",
       "      <th>url</th>\n",
       "      <th>publisher</th>\n",
       "      <th>address</th>\n",
       "      <th>year</th>\n",
       "      <th>...</th>\n",
       "      <th>doi</th>\n",
       "      <th>number</th>\n",
       "      <th>volume</th>\n",
       "      <th>journal</th>\n",
       "      <th>editor</th>\n",
       "      <th>isbn</th>\n",
       "      <th>ENTRYTYPE</th>\n",
       "      <th>ID</th>\n",
       "      <th>language</th>\n",
       "      <th>note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O02-2002</td>\n",
       "      <td>There is a need to measure word similarity whe...</td>\n",
       "      <td>There is a need to measure word similarity whe...</td>\n",
       "      <td>18022704</td>\n",
       "      <td>0b09178ac8d17a92f16140365363d8df88c757d0</td>\n",
       "      <td>14</td>\n",
       "      <td>https://aclanthology.org/O02-2002</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2002</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>chen-you-2002-study</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L02-1310</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>8220988</td>\n",
       "      <td>8d5e31610bc82c2abc86bc20ceba684c97e66024</td>\n",
       "      <td>93</td>\n",
       "      <td>http://www.lrec-conf.org/proceedings/lrec2002/...</td>\n",
       "      <td>European Language Resources Association (ELRA)</td>\n",
       "      <td>Las Palmas, Canary Islands - Spain</td>\n",
       "      <td>2002</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>mihalcea-2002-bootstrapping</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>R13-1042</td>\n",
       "      <td>Thread disentanglement is the task of separati...</td>\n",
       "      <td>Thread disentanglement is the task of separati...</td>\n",
       "      <td>16703040</td>\n",
       "      <td>3eb736b17a5acb583b9a9bd99837427753632cdb</td>\n",
       "      <td>10</td>\n",
       "      <td>https://aclanthology.org/R13-1042</td>\n",
       "      <td>INCOMA Ltd. Shoumen, BULGARIA</td>\n",
       "      <td>Hissar, Bulgaria</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>jamison-gurevych-2013-headerless</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>W05-0819</td>\n",
       "      <td>In this paper, we describe a word alignment al...</td>\n",
       "      <td>In this paper, we describe a word alignment al...</td>\n",
       "      <td>1215281</td>\n",
       "      <td>b20450f67116e59d1348fc472cfc09f96e348f55</td>\n",
       "      <td>15</td>\n",
       "      <td>https://aclanthology.org/W05-0819</td>\n",
       "      <td>Association for Computational Linguistics</td>\n",
       "      <td>Ann Arbor, Michigan</td>\n",
       "      <td>2005</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>aswani-gaizauskas-2005-aligning</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L02-1309</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>18078432</td>\n",
       "      <td>011e943b64a78dadc3440674419821ee080f0de3</td>\n",
       "      <td>12</td>\n",
       "      <td>http://www.lrec-conf.org/proceedings/lrec2002/...</td>\n",
       "      <td>European Language Resources Association (ELRA)</td>\n",
       "      <td>Las Palmas, Canary Islands - Spain</td>\n",
       "      <td>2002</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>suyaga-etal-2002-proposal</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     acl_id                                           abstract  \\\n",
       "0  O02-2002  There is a need to measure word similarity whe...   \n",
       "1  L02-1310                                               None   \n",
       "2  R13-1042  Thread disentanglement is the task of separati...   \n",
       "3  W05-0819  In this paper, we describe a word alignment al...   \n",
       "4  L02-1309                                               None   \n",
       "\n",
       "                                           full_text  corpus_paper_id  \\\n",
       "0  There is a need to measure word similarity whe...         18022704   \n",
       "1                                               None          8220988   \n",
       "2  Thread disentanglement is the task of separati...         16703040   \n",
       "3  In this paper, we describe a word alignment al...          1215281   \n",
       "4                                               None         18078432   \n",
       "\n",
       "                                   pdf_hash  numcitedby  \\\n",
       "0  0b09178ac8d17a92f16140365363d8df88c757d0          14   \n",
       "1  8d5e31610bc82c2abc86bc20ceba684c97e66024          93   \n",
       "2  3eb736b17a5acb583b9a9bd99837427753632cdb          10   \n",
       "3  b20450f67116e59d1348fc472cfc09f96e348f55          15   \n",
       "4  011e943b64a78dadc3440674419821ee080f0de3          12   \n",
       "\n",
       "                                                 url  \\\n",
       "0                  https://aclanthology.org/O02-2002   \n",
       "1  http://www.lrec-conf.org/proceedings/lrec2002/...   \n",
       "2                  https://aclanthology.org/R13-1042   \n",
       "3                  https://aclanthology.org/W05-0819   \n",
       "4  http://www.lrec-conf.org/proceedings/lrec2002/...   \n",
       "\n",
       "                                        publisher  \\\n",
       "0                                            None   \n",
       "1  European Language Resources Association (ELRA)   \n",
       "2                   INCOMA Ltd. Shoumen, BULGARIA   \n",
       "3       Association for Computational Linguistics   \n",
       "4  European Language Resources Association (ELRA)   \n",
       "\n",
       "                              address  year  ...   doi number volume journal  \\\n",
       "0                                None  2002  ...  None   None   None    None   \n",
       "1  Las Palmas, Canary Islands - Spain  2002  ...  None   None   None    None   \n",
       "2                    Hissar, Bulgaria  2013  ...  None   None   None    None   \n",
       "3                 Ann Arbor, Michigan  2005  ...  None   None   None    None   \n",
       "4  Las Palmas, Canary Islands - Spain  2002  ...  None   None   None    None   \n",
       "\n",
       "  editor  isbn      ENTRYTYPE                                ID language  note  \n",
       "0   None  None  inproceedings               chen-you-2002-study     None  None  \n",
       "1   None  None  inproceedings       mihalcea-2002-bootstrapping     None  None  \n",
       "2   None  None  inproceedings  jamison-gurevych-2013-headerless     None  None  \n",
       "3   None  None  inproceedings   aswani-gaizauskas-2005-aligning     None  None  \n",
       "4   None  None  inproceedings         suyaga-etal-2002-proposal     None  None  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_parquet('data/acl-publication-info.74k.v2.parquet')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\"acl_id\": \"paper_ident\", # unique paper identifier\n",
    "                   \"url\": \"paper_url\", # Paper online abstract page URL.\n",
    "                   \"author\": \"paper_author\", # Author list.\n",
    "                   \"title\": \"paper_title\", # Paper title\n",
    "                   \"journal\": \"paper_venue\", # Venue abbreviation.\n",
    "                   \"year\": \"paper_year\", # Publication year\n",
    "                   \"month\": \"paper_month\", # Publication month.\n",
    "                   \"booktitle\": \"paper_booktitle\", # BibTeX booktitle field.\n",
    "                   \"address\": \"paper_address\", # BibTeX adress field\n",
    "                   \"publisher\": \"paper_publisher\", # BibTeX publisher field      \n",
    "                   \"pages\": \"paper_pages\", # BibTeX pages.\n",
    "                   \"full_text\": \"paper_text\",\n",
    "                   })\n",
    "\n",
    "df = df.drop(columns=[\"abstract\", \"corpus_paper_id\", \"pdf_hash\", \"doi\",\n",
    "                              \"numcitedby\", \"number\", \"volume\",  \n",
    "                              \"editor\", \"isbn\", \"ENTRYTYPE\",\"ID\", \"language\", \"note\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "error_download\n",
       "False    67414\n",
       "True      5871\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['error_download'] = df['paper_text'].apply(lambda x: not x.strip() if isinstance(x, str) else True)\n",
    "\n",
    "df['error_download'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BLEU\n",
    "## Reproducibility\n",
    "### BLEU identification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_bleu_prelim\n",
       "False    58333\n",
       "True      9122\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_prelim = df.copy()\n",
    "df_bleu_prelim[\"paper_bleu_prelim\"] = df_bleu_prelim[\"paper_text\"].str.contains(\"bleu\", case=False)\n",
    "\n",
    "df_bleu_prelim['paper_bleu_prelim'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Paper Review\n",
    "#### BLEU Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_parameters(text):\n",
    "    pattern = r\"((?: -[a-z123](?: [a-z0-9.]{1,4})?){2,})\"\n",
    "    matches = re.findall(pattern, text)\n",
    "    return matches[0] if matches else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_params = df_bleu_prelim.copy()\n",
    "df_bleu_params[\"paper_bleu_params\"] = df_bleu_params.apply(lambda row: extract_parameters(row['paper_text']) if row['paper_bleu_prelim'] else None, axis=1)\n",
    "\n",
    "df_bleu_params['paper_bleu_params'].notna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BLEU Protocol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "bleu_regex_protocol = {\n",
    "    'ngrams': r\"\\bn-?grams?\\b\",\n",
    "    'precision': r\"\\bn-?gram\\sprecision\\b\",\n",
    "    'clipping': r\"\\bclipping\\b\",\n",
    "    'brevity_penalty': r\"\\bbrevity\\spenalty\\b|\\bBP\\b\",\n",
    "    'weights': r\"\\bweighting\\sof\\sn-?grams\\b\",\n",
    "    'smoothing': r\"\\bsmoothing\\b\",\n",
    "    'tokenization': r'\\b(?:tokenized?|tokenizer|tokenization|pre-tokenized?|detokenized?)\\b',\n",
    "    'case_normalization': r'\\b(?:case normalization|lowercased|case-insensitive|case sensitive)\\b'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_terms_near_bleu(text, regex_dict):\n",
    "    results = []\n",
    "    for term, pattern in regex_dict.items():\n",
    "        # Find all occurrences of 'bleu' (case insensitive)\n",
    "        for match in re.finditer(r'bleu', text, re.IGNORECASE):\n",
    "            start, end = match.start(), match.end()\n",
    "            # Define a 500-character window around 'bleu'\n",
    "            window_start, window_end = max(0, start - 500), min(len(text), end + 500)\n",
    "            # Search for the term within this window\n",
    "            if re.search(pattern, text[window_start:window_end], re.IGNORECASE):\n",
    "                results.append(term)\n",
    "    return list(set(results))  # Return unique terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_bleu_protocol\n",
       "[]                                            5297\n",
       "[ngrams]                                      1271\n",
       "[tokenization]                                 655\n",
       "[smoothing]                                    313\n",
       "[case_normalization]                           256\n",
       "[precision, ngrams]                            147\n",
       "[case_normalization, tokenization]             124\n",
       "[smoothing, ngrams]                            123\n",
       "[brevity_penalty, ngrams]                      123\n",
       "[tokenization, ngrams]                         120\n",
       "[brevity_penalty, precision, ngrams]            90\n",
       "[smoothing, tokenization]                       83\n",
       "[brevity_penalty]                               75\n",
       "[case_normalization, ngrams]                    65\n",
       "[case_normalization, smoothing]                 47\n",
       "[clipping]                                      33\n",
       "[case_normalization, smoothing, ngrams]         28\n",
       "[case_normalization, tokenization, ngrams]      28\n",
       "[tokenization, smoothing, ngrams]               28\n",
       "[brevity_penalty, smoothing, ngrams]            19\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_protocol=df_bleu_params.copy()\n",
    "df_bleu_protocol['paper_bleu_protocol'] = df_bleu_protocol[df_bleu_protocol['paper_bleu_prelim'] == True]['paper_text'].apply(lambda x: search_terms_near_bleu(x, bleu_regex_protocol))\n",
    "\n",
    "df_bleu_protocol[\"paper_bleu_protocol\"].value_counts().head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BLEU Variants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "bleu_regex_variants = {\n",
    "    'n_gram_precision': r'\\b(?:n-?gram precision|1-gram precision|2-gram precision|3-gram precision|4-gram precision)\\b',\n",
    "    'brevity_penalty': r'\\bbrevity penalty\\b|BP\\b'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_variants\n",
       "[]                                     8491\n",
       "[brevity_penalty]                       295\n",
       "[n_gram_precision]                      199\n",
       "[brevity_penalty, n_gram_precision]     137\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_variants=df_bleu_protocol.copy()\n",
    "df_bleu_variants['paper_variants'] = df_bleu_variants[df_bleu_variants['paper_bleu_prelim'] == True]['paper_text'].apply(lambda x: search_terms_near_bleu(x, bleu_regex_variants))\n",
    "\n",
    "df_bleu_variants['paper_variants'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BLEU Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_for_regex_pattern(text, regex_dict):\n",
    "    results = []\n",
    "    for term, pattern in regex_dict.items():\n",
    "        # Search for the pattern in the entire text\n",
    "        if re.search(pattern, text, re.IGNORECASE):\n",
    "            results.append(term)\n",
    "    return list(set(results))  # Return unique terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_bleu_versions = {\n",
    "    'BLEU_original': r'\\bBLEU\\b.*?Papineni.*?ACL.*?2002',\n",
    "    'multi_bleu': r'multi-?bleu|multi_bleu',\n",
    "    'sacreBLEU': r'sacrebleu',\n",
    "    'nltk_bleu': r'nltk.*?bleu',\n",
    "    'mteval_v13a': r'mteval_v?13a',\n",
    "    'mteval_v14': r'mteval_v?14',\n",
    "    'BLEU_moses': r'bleu.*?moses',\n",
    "    'BLEU_nematus': r'bleu.*?nematus',\n",
    "    'BLEU_coco': r'bleu.*?coco',\n",
    "    'BLEU_pytorch': r'bleu.*?pytorch',\n",
    "    'BLEU_tensorflow': r'bleu.*?tensorflow',\n",
    "    'BLEU_fairseq': r'fairseq.*?bleu',\n",
    "    'BLEU_sacremoses': r'sacremoses.*?bleu',\n",
    "    'nematus_bleu': r'nematus.*?bleu',\n",
    "    'subword_nmt_bleu': r'subword-?nmt.*?bleu',\n",
    "    'sentence_bleu': r'sentence-?bleu',\n",
    "    'corpus_bleu': r'corpus-?bleu',\n",
    "    'smoothing_bleu': r'smoothing.*?bleu',\n",
    "    \"coco\": r'coco.*?bleu',\n",
    "    \"pybleu\": r'pybleu|py-bleu',\n",
    "    \"google_bleu\": r'google.*?bleu',\n",
    "    \"yisi_bleu\": r'yisi.*?bleu',\n",
    "    \"bertscore_bleu\": r'bertscore.*?bleu',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_bleu_packages\n",
       "[]                                           3786\n",
       "[BLEU_moses]                                 1023\n",
       "[smoothing_bleu]                              672\n",
       "[google_bleu]                                 484\n",
       "[smoothing_bleu, BLEU_moses]                  482\n",
       "[BLEU_moses, google_bleu]                     144\n",
       "[sacreBLEU]                                   144\n",
       "[BLEU_pytorch]                                141\n",
       "[BLEU_fairseq]                                135\n",
       "[bertscore_bleu]                              112\n",
       "[nltk_bleu]                                    93\n",
       "[BLEU_moses, multi_bleu]                       84\n",
       "[coco, BLEU_coco]                              79\n",
       "[smoothing_bleu, google_bleu]                  53\n",
       "[multi_bleu]                                   50\n",
       "[BLEU_fairseq, sacreBLEU]                      49\n",
       "[smoothing_bleu, BLEU_fairseq]                 48\n",
       "[BLEU_original]                                47\n",
       "[smoothing_bleu, BLEU_moses, google_bleu]      46\n",
       "[BLEU_tensorflow]                              46\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_packages = df_bleu_variants.copy()\n",
    "# Applying the function to the DataFrame\n",
    "df_bleu_packages['paper_bleu_packages'] = df_bleu_packages[df_bleu_packages['paper_bleu_prelim'] == True]\\\n",
    "    ['paper_text'].apply(lambda x: search_for_regex_pattern(x, regex_bleu_versions))\n",
    "    \n",
    "df_bleu_packages[\"paper_bleu_packages\"].value_counts().head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code Review\n",
    "#### URL of code repository cited in paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bleu_url = df_bleu_packages.copy()\n",
    "\n",
    "# regex for codebases\n",
    "regex_codebases = r'https?://(?:www\\.)?(?:github\\.com|gitlab\\.com|bitbucket\\.org|sourceforge\\.net|google\\.code|code\\.google)[^\\s)]*(?<!\\.)'\n",
    "\n",
    "# Function to extract URLs from a text\n",
    "def extract_codebases(text):\n",
    "    return re.findall(regex_codebases, text)\n",
    "\n",
    "# Apply extract_codebases function to 'paper_text', store URLs in a list within each cell\n",
    "df_bleu_url[\"code_bleu_url\"] = df_bleu_url.apply(\n",
    "    lambda row: extract_codebases(row['paper_text']) if row['paper_bleu_prelim'] and pd.notnull(row['paper_text']) else [],\n",
    "    axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "code_bleu_url\n",
       "[]                                                                                        72028\n",
       "[https://github.com/]                                                                       175\n",
       "[https://github.com/pytorch/fairseq]                                                         18\n",
       "[https://github.com/moses-smt/]                                                              14\n",
       "[https://github.com/facebookresearch/]                                                       13\n",
       "                                                                                          ...  \n",
       "[https://github.com/facebookresearch/DPR#]                                                    1\n",
       "[https://github.com/cfedermann/]                                                              1\n",
       "[https://github.com/moses-smt/mosesdecoder, https://github.com/facebookresearch/MIXER]        1\n",
       "[https://github.com/fuzihaofzh/]                                                              1\n",
       "[https://github.com/alvations/]                                                               1\n",
       "Name: count, Length: 909, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_url[\"code_bleu_url\"].value_counts() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Does the code mention BLEU?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract GitHub repository names from URLs\n",
    "def extract_github_repo_names(urls):\n",
    "    return [\"/\".join(urlparse(url).path.strip(\"/\").split(\"/\")[:2]) for url in urls if \"github.com\" in urlparse(url).netloc]\n",
    "\n",
    "# Apply the function to extract GitHub repository names only if 'paper_bleu_prelim' is True\n",
    "df_bleu_url['code_bleu_github'] = df_bleu_url.apply(lambda row: extract_github_repo_names(row['code_bleu_url']) if row['paper_bleu_prelim'] else [], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "code_bleu_github\n",
       "[]                          72083\n",
       "[]                            182\n",
       "[pytorch/fairseq]              29\n",
       "[moses-smt]                    14\n",
       "[moses-smt/mosesdecoder]       13\n",
       "                            ...  \n",
       "[SimengSun]                     1\n",
       "[MIProtick]                     1\n",
       "[zhijing-jin/genwiki]           1\n",
       "[Marsan-Ma/chat]                1\n",
       "[alvations]                     1\n",
       "Name: count, Length: 841, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bleu_url['code_bleu_github'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bleu_reproducible = df_bleu_url.copy()\n",
    "\n",
    "# Initialize the 'reproducible' column as a nullable boolean\n",
    "df_bleu_reproducible['reproducible'] = pd.NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reproducible\n",
       "True    54\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# R1: Check if both packages and params are not null\n",
    "condition_r1 = df_bleu_reproducible['paper_bleu_packages'].notna() & df_bleu_reproducible['paper_bleu_params'].notna()\n",
    "df_bleu_reproducible.loc[condition_r1, 'reproducible'] = True\n",
    "\n",
    "df_bleu_reproducible['reproducible'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bleu_reproducible.to_pickle(\"bleu_paper_review.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_dataset(df: pd.DataFrame, filepath: str = \"meteorscores/data/bleu_papers.jsonl.gz\") -> None:\n",
    "    \"\"\"\n",
    "    Save the DataFrame to a .jsonl.gz file.\n",
    "    \n",
    "    Parameters:\n",
    "    - df: The DataFrame to save.\n",
    "    - filepath: The file path where the DataFrame should be saved.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        df.to_json(filepath, orient=\"records\", lines=True, compression=\"gzip\")\n",
    "        print(f\"Dataset successfully saved to {filepath}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Could not save dataset: {e}\")\n",
    "\n",
    "save_dataset(df_bleu_reproducible, \"meteorscores/data/bleu_papers.jsonl.gz\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}