METEOR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab Rotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\slim_\\miniconda3\\envs\\labrotation\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re as re\n",
    "import numpy as np\n",
    "import requests\n",
    "from urllib.parse import urlparse\n",
    "from time import sleep\n",
    "import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO \n",
    "github_token = \"ADD_YOUR_GITHUB_TOKEN\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Collection\n",
    "@Misc{acl-ocl,\n",
    "    author =       {Shaurya Rohatgi, Yanxia Qin, Benjamin Aw, Niranjana Unnithan, Min-Yen Kan},\n",
    "    title =        {The ACL OCL Corpus: advancing Open science in Computational Linguistics},\n",
    "    howpublished = {arXiv},\n",
    "    year =         {2022},\n",
    "    url =          {https://huggingface.co/datasets/ACL-OCL/ACL-OCL-Corpus}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acl_id</th>\n",
       "      <th>abstract</th>\n",
       "      <th>full_text</th>\n",
       "      <th>corpus_paper_id</th>\n",
       "      <th>pdf_hash</th>\n",
       "      <th>numcitedby</th>\n",
       "      <th>url</th>\n",
       "      <th>publisher</th>\n",
       "      <th>address</th>\n",
       "      <th>year</th>\n",
       "      <th>...</th>\n",
       "      <th>doi</th>\n",
       "      <th>number</th>\n",
       "      <th>volume</th>\n",
       "      <th>journal</th>\n",
       "      <th>editor</th>\n",
       "      <th>isbn</th>\n",
       "      <th>ENTRYTYPE</th>\n",
       "      <th>ID</th>\n",
       "      <th>language</th>\n",
       "      <th>note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O02-2002</td>\n",
       "      <td>There is a need to measure word similarity whe...</td>\n",
       "      <td>There is a need to measure word similarity whe...</td>\n",
       "      <td>18022704</td>\n",
       "      <td>0b09178ac8d17a92f16140365363d8df88c757d0</td>\n",
       "      <td>14</td>\n",
       "      <td>https://aclanthology.org/O02-2002</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2002</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>chen-you-2002-study</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L02-1310</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>8220988</td>\n",
       "      <td>8d5e31610bc82c2abc86bc20ceba684c97e66024</td>\n",
       "      <td>93</td>\n",
       "      <td>http://www.lrec-conf.org/proceedings/lrec2002/...</td>\n",
       "      <td>European Language Resources Association (ELRA)</td>\n",
       "      <td>Las Palmas, Canary Islands - Spain</td>\n",
       "      <td>2002</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>mihalcea-2002-bootstrapping</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>R13-1042</td>\n",
       "      <td>Thread disentanglement is the task of separati...</td>\n",
       "      <td>Thread disentanglement is the task of separati...</td>\n",
       "      <td>16703040</td>\n",
       "      <td>3eb736b17a5acb583b9a9bd99837427753632cdb</td>\n",
       "      <td>10</td>\n",
       "      <td>https://aclanthology.org/R13-1042</td>\n",
       "      <td>INCOMA Ltd. Shoumen, BULGARIA</td>\n",
       "      <td>Hissar, Bulgaria</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>jamison-gurevych-2013-headerless</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>W05-0819</td>\n",
       "      <td>In this paper, we describe a word alignment al...</td>\n",
       "      <td>In this paper, we describe a word alignment al...</td>\n",
       "      <td>1215281</td>\n",
       "      <td>b20450f67116e59d1348fc472cfc09f96e348f55</td>\n",
       "      <td>15</td>\n",
       "      <td>https://aclanthology.org/W05-0819</td>\n",
       "      <td>Association for Computational Linguistics</td>\n",
       "      <td>Ann Arbor, Michigan</td>\n",
       "      <td>2005</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>aswani-gaizauskas-2005-aligning</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L02-1309</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>18078432</td>\n",
       "      <td>011e943b64a78dadc3440674419821ee080f0de3</td>\n",
       "      <td>12</td>\n",
       "      <td>http://www.lrec-conf.org/proceedings/lrec2002/...</td>\n",
       "      <td>European Language Resources Association (ELRA)</td>\n",
       "      <td>Las Palmas, Canary Islands - Spain</td>\n",
       "      <td>2002</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>suyaga-etal-2002-proposal</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     acl_id                                           abstract  \\\n",
       "0  O02-2002  There is a need to measure word similarity whe...   \n",
       "1  L02-1310                                               None   \n",
       "2  R13-1042  Thread disentanglement is the task of separati...   \n",
       "3  W05-0819  In this paper, we describe a word alignment al...   \n",
       "4  L02-1309                                               None   \n",
       "\n",
       "                                           full_text  corpus_paper_id  \\\n",
       "0  There is a need to measure word similarity whe...         18022704   \n",
       "1                                               None          8220988   \n",
       "2  Thread disentanglement is the task of separati...         16703040   \n",
       "3  In this paper, we describe a word alignment al...          1215281   \n",
       "4                                               None         18078432   \n",
       "\n",
       "                                   pdf_hash  numcitedby  \\\n",
       "0  0b09178ac8d17a92f16140365363d8df88c757d0          14   \n",
       "1  8d5e31610bc82c2abc86bc20ceba684c97e66024          93   \n",
       "2  3eb736b17a5acb583b9a9bd99837427753632cdb          10   \n",
       "3  b20450f67116e59d1348fc472cfc09f96e348f55          15   \n",
       "4  011e943b64a78dadc3440674419821ee080f0de3          12   \n",
       "\n",
       "                                                 url  \\\n",
       "0                  https://aclanthology.org/O02-2002   \n",
       "1  http://www.lrec-conf.org/proceedings/lrec2002/...   \n",
       "2                  https://aclanthology.org/R13-1042   \n",
       "3                  https://aclanthology.org/W05-0819   \n",
       "4  http://www.lrec-conf.org/proceedings/lrec2002/...   \n",
       "\n",
       "                                        publisher  \\\n",
       "0                                            None   \n",
       "1  European Language Resources Association (ELRA)   \n",
       "2                   INCOMA Ltd. Shoumen, BULGARIA   \n",
       "3       Association for Computational Linguistics   \n",
       "4  European Language Resources Association (ELRA)   \n",
       "\n",
       "                              address  year  ...   doi number volume journal  \\\n",
       "0                                None  2002  ...  None   None   None    None   \n",
       "1  Las Palmas, Canary Islands - Spain  2002  ...  None   None   None    None   \n",
       "2                    Hissar, Bulgaria  2013  ...  None   None   None    None   \n",
       "3                 Ann Arbor, Michigan  2005  ...  None   None   None    None   \n",
       "4  Las Palmas, Canary Islands - Spain  2002  ...  None   None   None    None   \n",
       "\n",
       "  editor  isbn      ENTRYTYPE                                ID language  note  \n",
       "0   None  None  inproceedings               chen-you-2002-study     None  None  \n",
       "1   None  None  inproceedings       mihalcea-2002-bootstrapping     None  None  \n",
       "2   None  None  inproceedings  jamison-gurevych-2013-headerless     None  None  \n",
       "3   None  None  inproceedings   aswani-gaizauskas-2005-aligning     None  None  \n",
       "4   None  None  inproceedings         suyaga-etal-2002-proposal     None  None  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_parquet('data/acl-publication-info.74k.v2.parquet')\n",
    "#df2 = datasets.load_dataset(\"WINGNUS/ACL-OCL\")\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\"acl_id\": \"paper_ident\", # unique paper identifier\n",
    "                   \"url\": \"paper_url\", # Paper online abstract page URL.\n",
    "                   \"author\": \"paper_author\", # Author list.\n",
    "                   \"title\": \"paper_title\", # Paper title\n",
    "                   \"journal\": \"paper_venue\", # Venue abbreviation.\n",
    "                   \"year\": \"paper_year\", # Publication year\n",
    "                   \"month\": \"paper_month\", # Publication month.\n",
    "                   \"booktitle\": \"paper_booktitle\", # BibTeX booktitle field.\n",
    "                   \"address\": \"paper_address\", # BibTeX adress field\n",
    "                   \"publisher\": \"paper_publisher\", # BibTeX publisher field      \n",
    "                   \"pages\": \"paper_pages\", # BibTeX pages.\n",
    "                   \"full_text\": \"paper_text\",\n",
    "                   })\n",
    "\n",
    "df = df.drop(columns=[\"abstract\", \"corpus_paper_id\", \"pdf_hash\", \"doi\",\n",
    "                              \"numcitedby\", \"number\", \"volume\",  \n",
    "                              \"editor\", \"isbn\", \"ENTRYTYPE\",\"ID\", \"language\", \"note\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "error_download\n",
       "False    67414\n",
       "True      5871\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['error_download'] = df['paper_text'].apply(lambda x: not x.strip() if isinstance(x, str) else True)\n",
    "\n",
    "df['error_download'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Meteor Scores\n",
    "## Reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### METEOR Identification - paper_meteor_prelim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_meteor_prelim\n",
       "False    65842\n",
       "True      1613\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meteor_prelim = df.copy()\n",
    "df_meteor_prelim[\"paper_meteor_prelim\"] = df_meteor_prelim[\"paper_text\"].str.contains(\" meteor \", case=False)\n",
    "\n",
    "df_meteor_prelim['paper_meteor_prelim'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Paper Review\n",
    "#### METEOR Parameters - paper_meteor_params\n",
    "\n",
    "\n",
    "Parameters according to https://www.cs.cmu.edu/~alavie/METEOR/README.html\n",
    "\n",
    "-l language                     Fully supported: en cz de es fr ar\n",
    "                                Supported with language-independent parameters:\n",
    "                                  da fi hu it nl no pt ro ru se \n",
    "                                  \n",
    "-t task                         One of: rank util adq hter li tune\n",
    "                                  util implies -ch\n",
    "\n",
    "-p 'alpha beta gamma delta'     Custom parameters (overrides default)\n",
    "\n",
    "-m 'module1 module2 ...'        Specify modules (overrides default)\n",
    "                                  Any of: exact stem synonym paraphrase\n",
    "\n",
    "-w 'weight1 weight2 ...'        Specify module weights (overrides default)\n",
    "\n",
    "-r refCount                     Number of references (plaintext only)\n",
    "\n",
    "-x beamSize                     (default 40)\n",
    "\n",
    "-s wordListDirectory            (if not default for language)\n",
    "\n",
    "-d synonymDirectory             (if not default for language)\n",
    "\n",
    "-a paraphraseFile               (if not default for language)\n",
    "\n",
    "-f filePrefix                   Prefix for output files (default 'meteor')\n",
    "\n",
    "-q                              Quiet: Segment scores to stderr, final to stdout,\n",
    "                                  no additional output (plaintext only)\n",
    "\n",
    "-ch                             Character-based precision and recall\n",
    "\n",
    "-norm                           Tokenize / normalize punctuation and lowercase\n",
    "                                  (Recommended unless scoring raw output with\n",
    "                                   pretokenized references)\n",
    "\n",
    "-lower                          Lowercase only (not required if -norm specified)\n",
    "\n",
    "-noPunct                        Do not consider punctuation when scoring\n",
    "                                  (Not recommended unless special case)\n",
    "\n",
    "-sgml                           Input is in SGML format\n",
    "\n",
    "-mira                           Input is in MIRA format\n",
    "                                  (Use '-' for test and reference files)\n",
    "\n",
    "-vOut                           Output verbose scores (P / R / frag / score)\n",
    "\n",
    "-ssOut                          Output sufficient statistics instead of scores\n",
    "\n",
    "-writeAlignments                Output alignments annotated with Meteor scores\n",
    "                                  (written to <prefix>-align.out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_parameters(text):\n",
    "    pattern = r\"((?: -[a-z123](?: [a-z0-9.]{1,4})?){2,})\"\n",
    "    matches = re.findall(pattern, text)\n",
    "    return matches[0] if matches else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_meteor_params\n",
       " -s or -e                          2\n",
       " -1 -1                             2\n",
       " -o -v                             1\n",
       " -3 to -2                          1\n",
       " -n 2 -m -u -c 95 -x -r 1000 -f    1\n",
       " -t -d 0                           1\n",
       " -e -a pojk                        1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meteor_params = df_meteor_prelim.copy()\n",
    "df_meteor_params[\"paper_meteor_params\"] = df_meteor_params.apply(lambda row: extract_parameters(row['paper_text']) if row['paper_meteor_prelim'] else None, axis=1)\n",
    "\n",
    "df_meteor_params['paper_meteor_params'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### METEOR Protocol - paper_meteor_protocol\n",
    "- Language: -l language\n",
    "    - Use settings for specified language. Lang can be either the language name or two letter code. See the supported language list.\n",
    "    - Fully supported: en cz de es fr ar\n",
    "    - Supported with language-independent parameters: fi hu it nl no pt ro ru se tr\n",
    "    \n",
    "- Task: -t task\n",
    "    - Use a different pre-defined set of parameters for scoring (currently limited to English):\n",
    "        - rank: parameters tuned to human rankings from WMT09 and WMT10\n",
    "        - adq: parameters tuned to adequacy scores from NIST Open MT 2009\n",
    "        - hter: parameters tuned to HTER scores from GALE P2 and P3\n",
    "        - li: language-independent parameters\n",
    "    - One of: rank util adq hter li tune (util implies -ch)\n",
    "\n",
    "- Modules: -m 'module1 module2 ...'\n",
    "    - Set modules manually. Options are: exact stem synonym paraphrase. See supported languages. Module string should be quoted. \n",
    "\n",
    "- Normalize: -norm\n",
    "    - Tokenize and lowercases input lines, normalize punctuation to improve scoring accuracy. This option is highly recommended unless scoring raw system output against pretokenized references. \n",
    "\n",
    "- Lowercase: -lower\n",
    "    - Lowercase input lines (not required if -norm also specified). This is most commonly used scoring cased, tokenized outputs with pretokenized references. \n",
    "\n",
    "- Ignore Punctuation: -noPunct\n",
    "    - If specified, punctuation symbols will be removed before scoring. This is generally not recommended as parameters are tuned with punctuation included. \n",
    "\n",
    "- Character-based -ch\n",
    "    - Calculate character-based precision and recall. Alignment is still word and phrase-level. Fragmentation penalty is still word and phrase-level. \n",
    "\n",
    "- Verbose Output: -vOut\n",
    "    - Output verbose scores (Precision, Recall, Fragmentation, Score) in place of regular scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_meteor_protocol = {\n",
    "    'rank': r'\\b(?:rank|ranking|WMT09|WMT10)\\b',\n",
    "    'adq': r'\\b(?:adq|adequacy|NIST Open MT 2009)\\b',\n",
    "    'hter': r'\\b(?:hter|HUMAN-targeted translation edit rate|GALE P2|GALE P3)\\b',\n",
    "    'li': r'\\b(?:li|language[- ]independent)\\b',\n",
    "    'modules': r'\\b(?:-m\\s+(?:exact|stem|synonym|paraphrase)|module|exact|stem|synonym|paraphrase)\\b',\n",
    "    'normalize': r'\\b(?:-norm|normalize|normalization|tokenize\\s+and\\s+lowercase|normalize\\s+punctuation)\\b',\n",
    "    'lowercase': r'\\b(?:-lower|lowercase|lowercasing|casing)\\b',\n",
    "    'ignore_punctuation': r'\\b(?:-noPunct|no\\s+Punctuation|ignore\\s+punctuation|remove\\s+punctuation)\\b',\n",
    "    'character_based': r'\\b(?:-ch|character[- ]based|calculate\\s+character[- ]based\\s+precision\\s+and\\s+recall)\\b',\n",
    "    'verbose_output': r'\\b(?:-vOut|verbose\\s+output|output\\s+verbose\\s+scores)\\b'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_terms_near_meteor(text, regex_dict):\n",
    "    results = []\n",
    "    for term, pattern in regex_dict.items():\n",
    "        # Find all occurrences of 'meteor' (case insensitive)\n",
    "        for match in re.finditer(r'meteor', text, re.IGNORECASE):\n",
    "            start, end = match.start(), match.end()\n",
    "            # Define a 500-character window around 'meteor'\n",
    "            window_start, window_end = max(0, start - 500), min(len(text), end + 500)\n",
    "            # Search for the term within this window\n",
    "            if re.search(pattern, text[window_start:window_end], re.IGNORECASE):\n",
    "                results.append(term)\n",
    "    return list(set(results))  # Return unique terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_meteor_protocol\n",
       "[]                            947\n",
       "[modules]                     211\n",
       "[rank]                         98\n",
       "[adq]                          62\n",
       "[li]                           53\n",
       "[modules, rank]                47\n",
       "[modules, li]                  23\n",
       "[adq, modules]                 20\n",
       "[adq, rank, modules]           13\n",
       "[adq, rank]                    12\n",
       "[hter]                         11\n",
       "[modules, normalize]           11\n",
       "[normalize]                    10\n",
       "[rank, li]                     10\n",
       "[lowercase]                     8\n",
       "[character_based]               8\n",
       "[adq, rank, hter, modules]      7\n",
       "[lowercase, modules]            6\n",
       "[modules, rank, li]             5\n",
       "[adq, modules, li]              4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meteor_protocol=df_meteor_params.copy()\n",
    "df_meteor_protocol['paper_meteor_protocol'] = df_meteor_protocol[df_meteor_protocol['paper_meteor_prelim'] == True]['paper_text'].apply(lambda x: search_terms_near_meteor(x, regex_meteor_protocol))\n",
    "\n",
    "df_meteor_protocol[\"paper_meteor_protocol\"].value_counts().head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### METEOR packages - paper_meteor_packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_for_regex_pattern(text, regex_dict):\n",
    "    results = []\n",
    "    for term, pattern in regex_dict.items():\n",
    "        # Search for the pattern in the entire text\n",
    "        if re.search(pattern, text, re.IGNORECASE):\n",
    "            results.append(term)\n",
    "    return list(set(results))  # Return unique terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_meteor_packages = {\n",
    "    # METEOR versions\n",
    "    # uncomment this to see an overlap between METEOR versions and packages\n",
    "    # 'Meteor_1.5': r'Meteor\\s+Universal.*?Denkowski.*?Lavie.*?EACL.*?2014',\n",
    "    # 'Meteor_1.3': r'Meteor\\s+1\\.3.*?Denkowski.*?Lavie.*?EMNLP.*?2011',\n",
    "    # 'Meteor_NEXT': r'METEOR(?:-NEXT)?.*?Denkowski.*?Lavie.*?ACL.*?2010',\n",
    "    # 'Meteor_Phrase_Level': r'Meteor.*?Phrase\\s+Level.*?NAACL(?:/HLT)?.*?2010',\n",
    "    # 'Meteor_MT_2010': r'METEOR.*?Machine\\s+Translation(?:\\s+Workshop)?.*?2010',\n",
    "    # 'Meteor_MBleu_MTer': r'METEOR.*?(?:M-BLEU|M-TER).*?ACL.*?2008',\n",
    "    # 'Meteor_Automatic_MT_2007': r'METEOR.*?Automatic.*?MT\\s+Evaluation.*?ACL.*?2007',\n",
    "    # 'Meteor_Automatic_MT_2005': r'METEOR.*?Automatic.*?MT\\s+Evaluation.*?ACL.*?2005',\n",
    "    # METEOR implementations\n",
    "    # identified by manual research and\n",
    "    # code repositories mentioned > 1 times (see code review)\n",
    "    'Meteor_coco': r'(?:coco.*?meteor|meteor.*?coco)',\n",
    "    'pymeteor': r'pymeteor|zembrodt',   \n",
    "    'generationeval_meteor': r'generationeval.*?meteor|meteor.*?generationeval|webnlg.*?meteor|meteor.*?webnlg',\n",
    "    'evaluatemetric_meteor': r'evaluatemetric.*?meteor|meteor.*?evaluate(?:-)?metric',    \n",
    "    'fairseq_meteor': r'fairseq.*?meteor|meteor.*?fairseq',\n",
    "    'nlgeval_meteor': r'nlgeval.*?meteor|meteor.*?nlg(?:-)?eval',\n",
    "    'nltk_meteor': r'nltk.*?meteor|meteor.*?nltk',\n",
    "    'meteorscorer': r'meteorscorer',\n",
    "    #'beer_meteor': r'beer.*?meteor|meteor.*?beer',\n",
    "    'compare_mt_meteor': r'compare(?:-)?mt.*?meteor|meteor.*?compare(?:-)?mt',\n",
    "    'pysimt_meteor': r'pysimt.*?meteor|meteor.*?pysimt',\n",
    "    #'blend_meteor': r'blend.*?meteor|meteor.*?blend',\n",
    "    'stasis_meteor': r'stasis.*?meteor|meteor.*?stasis',\n",
    "    'huggingface' : r'hugging\\s*face.*?evaluate.*?meteor|meteor.*?hugging\\s*face.*?evaluate',\n",
    "    'SummEval_meteor': r'SummEval.*?meteor|meteor.*?Yale-LILY|Yale-LILY.*?meteor|meteor.*?SummEval',\n",
    "    'vizseq_meteor': r'vizseq.*?meteor|meteor.*?vizseq|facebookresearch.*?meteor|meteor.*?facebookresearch',\n",
    "    'github meteor': r'github.com.{0,50}meteor',\n",
    "    # Template for other versions\n",
    "    'Meteor_x': r'Meteor\\s+[xX]\\.?[\\d\\.]*', \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paper_meteor_packages\n",
       "[]                                      1250\n",
       "[Meteor_coco]                            128\n",
       "[nltk_meteor]                             48\n",
       "[generationeval_meteor]                   44\n",
       "[huggingface]                             36\n",
       "[fairseq_meteor]                          28\n",
       "[Meteor_coco, fairseq_meteor]              9\n",
       "[nlgeval_meteor]                           8\n",
       "[SummEval_meteor]                          6\n",
       "[huggingface, nltk_meteor]                 5\n",
       "[huggingface, generationeval_meteor]       4\n",
       "[Meteor_coco, nlgeval_meteor]              4\n",
       "[Meteor_coco, nltk_meteor]                 4\n",
       "[generationeval_meteor, nltk_meteor]       4\n",
       "[huggingface, fairseq_meteor]              3\n",
       "[vizseq_meteor]                            2\n",
       "[Meteor_x]                                 2\n",
       "[compare_mt_meteor]                        2\n",
       "[Meteor_coco, generationeval_meteor]       2\n",
       "[compare_mt_meteor, fairseq_meteor]        2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meteor_packages = df_meteor_protocol.copy()\n",
    "# Applying the function to the DataFrame\n",
    "df_meteor_packages['paper_meteor_packages'] = df_meteor_packages[df_meteor_packages['paper_meteor_prelim'] == True]['paper_text'].apply(lambda x: search_for_regex_pattern(x, regex_meteor_packages))\n",
    "\n",
    "df_meteor_packages[\"paper_meteor_packages\"].value_counts().head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code Review\n",
    "#### URL of code repository cited in paper - code_meteor_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "code_meteor_url\n",
       "[]                                             73021\n",
       "[https://github.com/]                             35\n",
       "[http://code.google.com/p/giza-pp/]                4\n",
       "[https://github.com/google-research/]              3\n",
       "[https://github.com/UKPLab/]                       2\n",
       "                                               ...  \n",
       "[https://github.com/proycon/deepfrog]              1\n",
       "[https://github.com/QAQ-v/HetGT]                   1\n",
       "[https://github.com/c-w/gutenberg]                 1\n",
       "[https://github.com/google-research/pangea]        1\n",
       "[https://github.com/shiranD/word_]                 1\n",
       "Name: count, Length: 218, dtype: int64"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meteor_url = df_meteor_packages.copy()\n",
    "\n",
    "# Define regex for matching codebase URLs\n",
    "regex_codebases = r'https?://(?:www\\.)?(?:github\\.com|gitlab\\.com|bitbucket\\.org|sourceforge\\.net|google\\.code|code\\.google)[^\\s)]*(?<!\\.)'\n",
    "\n",
    "# Function to extract codebase URLs from text using regex\n",
    "def extract_codebases(text):\n",
    "    # Use re.findall to match all instances of the regex in the text\n",
    "    return re.findall(regex_codebases, text)\n",
    "\n",
    "# Apply extract_codebases function to 'paper_text', store URLs in a list within each cell\n",
    "df_meteor_url[\"code_meteor_url\"] = df_meteor_url.apply(\n",
    "    lambda row: extract_codebases(row['paper_text']) if row['paper_meteor_prelim'] and pd.notnull(row['paper_text']) else [],\n",
    "    axis=1\n",
    ")\n",
    "\n",
    "df_meteor_url[\"code_meteor_url\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Does the code mention METEOR? - code_meteor_prelim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract GitHub repository names from URLs\n",
    "def extract_github_repo_names(urls):\n",
    "    return [\"/\".join(urlparse(url).path.strip(\"/\").split(\"/\")[:2]) for url in urls if \"github.com\" in urlparse(url).netloc]\n",
    "\n",
    "# Apply the function to extract GitHub repository names only if 'paper_meteor_prelim' is True\n",
    "df_meteor_url['code_meteor_github'] = df_meteor_url.apply(lambda row: extract_github_repo_names(row['code_meteor_url']) if row['paper_meteor_prelim'] else [], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_code = df_meteor_url.copy()\n",
    "\n",
    "# Initialize column for search results\n",
    "df_meteor_code['code_meteor_prelim'] = False\n",
    "\n",
    "# Collect unique GitHub repositories from papers with 'paper_meteor_prelim' == True\n",
    "unique_repos = set(repo for repos_list in df_meteor_code[df_meteor_code['paper_meteor_prelim'] == True]['code_meteor_github'] for repo in repos_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_packages.to_pickle(\"meteor_paper_review.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting search for 'meteor' in 227 unique GitHub repositories...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "code_meteor_prelim\n",
       "False    71867\n",
       "True        56\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Search GitHub repositories for \"meteor\" keyword\n",
    "def search_github_repos(unique_repos, token):\n",
    "    print(f\"Starting search for 'meteor' in {len(unique_repos)} unique GitHub repositories...\")\n",
    "    repo_search_results = {}\n",
    "    headers = {'Authorization': f'token {token}', 'Accept': 'application/vnd.github.v3+json'}\n",
    "    for idx, repo in enumerate(unique_repos):\n",
    "        search_url = f\"https://api.github.com/search/code?q=meteor+repo:{repo}\"\n",
    "        try:\n",
    "            response = requests.get(search_url, headers=headers)\n",
    "            repo_search_results[repo] = response.status_code == 200 and response.json()['total_count'] > 0\n",
    "        except Exception as e:\n",
    "            print(f\"Error searching {repo}: {e}\")\n",
    "        if (idx + 1) % 10 == 0:\n",
    "            sleep(60)  # Sleep to respect the GitHub search API rate limit\n",
    "    return repo_search_results\n",
    "\n",
    "# Perform the search\n",
    "repo_search_results = search_github_repos(unique_repos, github_token)\n",
    "\n",
    "# Update 'code_meteor_prelim' based on the search results\n",
    "def update_prelim_column(repos):\n",
    "    if not repos:  # If the list of repositories is empty, do not change the value\n",
    "        return np.nan\n",
    "    return any(repo_search_results.get(repo, False) for repo in repos)\n",
    "\n",
    "# Apply the update function to 'code_meteor_prelim' only for rows where 'paper_meteor_prelim' is True\n",
    "df_meteor_code['code_meteor_prelim'] = df_meteor_code.apply(lambda row: update_prelim_column(row['code_meteor_github']) if row['paper_meteor_prelim'] else row['code_meteor_prelim'], axis=1)\n",
    "\n",
    "df_meteor_code['code_meteor_prelim'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_code.to_pickle(\"meteor_code.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manual Code Review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_code_review = df_meteor_code.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a separate DataFrame for GitHub repositories where \"meteor\" has been found\n",
    "df_meteor_code_github = pd.DataFrame([repo for repo, found in repo_search_results.items() if found], columns=['github_repository'])\n",
    "\n",
    "# Export the DataFrame with counts to a CSV file\n",
    "df_meteor_code_github.to_csv('meteor_github.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_code_manual = pd.read_csv(\"meteor_code_review.csv\", sep=\";\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "code_meteor\n",
      "True     33\n",
      "False    13\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Explode df_meteor_code to have one repository name per row for merging, ensure 'acl_id' is retained\n",
    "df_meteor_code_exploded = df_meteor_code.explode('code_meteor_github')\n",
    "\n",
    "# Merge the manual review data into the exploded dataframe on 'code_meteor_github'\n",
    "df_merged = pd.merge(\n",
    "    df_meteor_code_exploded,\n",
    "    df_meteor_code_manual[['code_meteor_github', 'code_meteor']],\n",
    "    on='code_meteor_github',\n",
    "    how='left'\n",
    ")\n",
    "\n",
    "# Define aggregation methods for all columns\n",
    "# 'first' for most columns to avoid duplication, 'max' for 'code_meteor' assuming it's boolean or numerical\n",
    "aggregation_methods = {col: 'first' for col in df_merged.columns if col not in ['code_meteor_github', 'code_meteor']}\n",
    "aggregation_methods['code_meteor_github'] = lambda x: list(set(x.dropna()))  # Unique GitHub repository names\n",
    "aggregation_methods['code_meteor'] = 'max'  # Max to ensure if any true value exists, it prevails\n",
    "\n",
    "# Group by 'acl_id' and aggregate according to defined methods\n",
    "df_meteor_code_review = df_merged.groupby('paper_ident', as_index=False).agg(aggregation_methods)\n",
    "\n",
    "# Optional: Clean up 'code_meteor_github' if necessary, removing any potential empty strings\n",
    "df_meteor_code_review['code_meteor_github'] = df_meteor_code_review['code_meteor_github'].apply(lambda x: [item for item in x if item])\n",
    "\n",
    "print(df_meteor_code_review[\"code_meteor\"].value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Software error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_code_review['software_error'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "software_error\n",
       "False    73123\n",
       "True       162\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "packages_with_errors = ['nltk_meteor', 'fairseq_meteor', 'huggingface']\n",
    "\n",
    "# Function to check if any package in the list is contained in the paper_meteor_packages column\n",
    "def contains_package(packages):\n",
    "    # Check if 'packages' is not a list (e.g., NaN or None)\n",
    "    if not isinstance(packages, list):\n",
    "        return False\n",
    "    # Check if any of the specified packages with errors are in the 'packages' list\n",
    "    return any(pkg in packages for pkg in packages_with_errors)\n",
    "\n",
    "# Apply the function to the paper_meteor_packages column and assign the result to the software_error column\n",
    "df_meteor_code_review['software_error'] = df_meteor_packages['paper_meteor_packages'].apply(contains_package)\n",
    "\n",
    "df_meteor_code_review['software_error'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reproducibility\n",
    "R1: Paper cites METEOR package and parameters.\n",
    "\n",
    "R2: Paper cites no-config4 METEOR package.\n",
    "\n",
    "R3: Codebase has complete METEOR evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_reproducible = df_meteor_code_review.copy()\n",
    "\n",
    "# Initialize the 'reproducible' column as a nullable boolean\n",
    "df_meteor_reproducible['reproducible'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reproducible\n",
       "False    73276\n",
       "True         9\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# R1: Check if both packages and params are not null\n",
    "condition_r1 = df_meteor_reproducible['paper_meteor_packages'].notna() & df_meteor_reproducible['paper_meteor_params'].notna()\n",
    "df_meteor_reproducible.loc[condition_r1, 'reproducible'] = True\n",
    "\n",
    "df_meteor_reproducible['reproducible'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reproducible\n",
       "False    72928\n",
       "True       357\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# R2: Check for no configuration packages\n",
    "# Define the no configuration packages list\n",
    "no_config_packages = ['generationeval_meteor', 'Meteor_coco', 'pymeteor', 'nlgeval_meteor', 'nltk_meteor',\n",
    "                      'huggingface', 'fairseq_meteor', ]\n",
    "\n",
    "def check_reproducibility(row):\n",
    "    # Only modify if reproducible is False or pd.NA\n",
    "    if row['reproducible'] is False or pd.isna(row['reproducible']):\n",
    "        # Check if paper_meteor_packages is a list and not empty or NA\n",
    "        if isinstance(row['paper_meteor_packages'], list) and row['paper_meteor_packages']:\n",
    "            # Check if any package in the list requires no configuration\n",
    "            if any(pkg in no_config_packages for pkg in row['paper_meteor_packages']):\n",
    "                return True\n",
    "    return row['reproducible']\n",
    "\n",
    "# Apply the function to update the 'reproducible' column\n",
    "df_meteor_reproducible['reproducible'] = df_meteor_reproducible.apply(check_reproducibility, axis=1)\n",
    "\n",
    "df_meteor_reproducible['reproducible'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reproducible\n",
       "False    72911\n",
       "True       374\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# R3: Check if 'code_meteor' is True\n",
    "condition_r3 = df_meteor_reproducible['code_meteor'] == True\n",
    "df_meteor_reproducible.loc[condition_r3, 'reproducible'] = True\n",
    "\n",
    "df_meteor_reproducible['reproducible'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_reproducible[\"code_meteor\"] = df_meteor_reproducible[\"code_meteor\"].apply(lambda x: False if x != True else True).fillna(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_meteor_reproducible.to_pickle(\"meteor_review.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset successfully saved to meteorscores/data/meteor_papers.jsonl.gz\n"
     ]
    }
   ],
   "source": [
    "def save_dataset(df: pd.DataFrame, filepath: str = \"meteorscores/data/meteor_papers.jsonl.gz\") -> None:\n",
    "    \"\"\"\n",
    "    Save the DataFrame to a .jsonl.gz file, excluding the 'paper_text' column.\n",
    "    \n",
    "    Parameters:\n",
    "    - df: The DataFrame to save.\n",
    "    - filepath: The file path where the DataFrame should be saved.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        df.drop(columns=['paper_text']).to_json(filepath, orient=\"records\", lines=True, compression=\"gzip\")\n",
    "        print(f\"Dataset successfully saved to {filepath}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Could not save dataset: {e}\")\n",
    "\n",
    "save_dataset(df_meteor_reproducible, \"meteorscores/data/meteor_papers.jsonl.gz\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}