2_resolve_entity_level.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resolve the data to entity-level\n",
    "The data_cache now has the information we need, on the sentence level. We resolve the entities for each document, and record the sentiment scores for the document and sentence(s) they occur in. We also resolve any sentiment directed towards them at the target level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# venv transform\n",
    "import json\n",
    "import os\n",
    "import time\n",
    "from collections import Counter, defaultdict\n",
    "import numpy as np\n",
    "import random\n",
    "import pandas as pd\n",
    "import torch\n",
    "from helpers import *\n",
    "from transformers import  pipeline\n",
    "from tqdm import tqdm\n",
    "from itertools import product\n",
    "\n",
    "\n",
    "save_root = \"outputs/\"\n",
    "tabular_savefolder = os.path.join(save_root, \"tabular\")\n",
    "os.makedirs(tabular_savefolder, exist_ok=True)\n",
    "data_cache = os.path.join(tabular_savefolder, \"data_sentencewise.json\")\n",
    "with open(data_cache, encoding = \"utf-8\") as rf:\n",
    "    dataset = json.load(rf)\n",
    "inspect_ids = tuple(set([s[\"doc_id\"] for s in dataset]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resolve entities to document-level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resolve_sent_pols(polarities):\n",
    "    \"\"\"List of sentence polarities the entity appears in.\n",
    "    Most commom pos or neg wins. Mixed for tie\"\"\"\n",
    "    strengths = {}\n",
    "    for polarity in ['Positive', 'Negative', 'Neutral', \"\"]:\n",
    "        strengths[polarity] = len([p for p in polarities if p == polarity])\n",
    "        if strengths[polarity] == len(polarities):\n",
    "            return polarity\n",
    "    if strengths[\"Positive\"] > strengths[\"Negative\"]:\n",
    "        return \"Positive\"\n",
    "    if strengths[\"Negative\"] > strengths[\"Positive\"]:\n",
    "        return \"Negative\"\n",
    "    return \"Mixed\"\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_dfs = [] # List of document-entity dataframes to simplify the merging\n",
    "entity_dict = {}\n",
    "\n",
    "for doc_id in inspect_ids:\n",
    "    doc_ents = []\n",
    "    doc_sents = [s for s in dataset if s[\"doc_id\"] == doc_id]\n",
    "    ne_groups = [] # List of lists of groups\n",
    "    for sent in doc_sents:\n",
    "        for named_e in sent[\"nes\"]:\n",
    "            # Collect entity with substring match and keep enough data for later\n",
    "            ne = named_e[\"text\"]\n",
    "            found = False\n",
    "            for ne_group in ne_groups:\n",
    "                if any([ne in existing for existing in ne_group]) or any([existing in ne for existing in ne_group]):\n",
    "                    ne_group.append(ne)\n",
    "                    found = True\n",
    "                    break\n",
    "                else :\n",
    "                    stripped = ne.rstrip(\"s\").rstrip(\"'\").rstrip()\n",
    "                    if any([stripped in existing for existing in ne_group]):\n",
    "                        ne_group.append(ne)\n",
    "                        found = True\n",
    "                        break\n",
    "\n",
    "            if not found:\n",
    "                ne_groups.append([ne])\n",
    "        \n",
    "    # Now, each named entity document-level is a list of apperances in the text.\n",
    "    # Next step is to iterate these and make a dataframe\n",
    "    doc_entities = {} # Resolved name as key, all mentions in value\n",
    "    for s_forms in ne_groups:\n",
    "        # e_ent is a list of dicts representing each surface form of same entity\n",
    "        # Find longest text representat\n",
    "        longest = max(s_forms, key=len)\n",
    "        if longest.rstrip(\"s\").rstrip(\"'\").rstrip() in s_forms:\n",
    "            longest = longest.rstrip(\"s\").rstrip(\"'\").rstrip()\n",
    "        doc_entities[longest] = list(set(s_forms))\n",
    "\n",
    "        \n",
    "    # If we read the entities like [\"John\", \"Wayne\",  \"John Wayne\"] We get two lists because Wayne is not substring og John. Fixing this\n",
    "    for one, two in product(doc_entities.copy(), doc_entities.copy()):\n",
    "        if not one == two and one in two:\n",
    "            doc_entities[two] += doc_entities[one]\n",
    "            del doc_entities[one]\n",
    "    \n",
    "    # print(doc_id, doc_entities)\n",
    "\n",
    "    \"Double check no duplicate entries in different surface form lists\"\n",
    "    all_surface_forms = [f for  s_forms in doc_entities.values() for f in s_forms]\n",
    "    assert len(all_surface_forms) == len(set(all_surface_forms))\n",
    "\n",
    "    # Populate the entity with more data\n",
    "    for longest, s_forms in doc_entities.items():\n",
    "        doc_entities[longest] = {\"surface_forms\": s_forms,\n",
    "            \"doc_id\": doc_id,\n",
    "            \"entity_id\":  doc_id+\"_\"+\"_\".join(longest.split())\n",
    "        }\n",
    "\n",
    "    for longest, ent_data in doc_entities.copy().items():\n",
    "        sents_having = []\n",
    "        nes_belonging = []\n",
    "        for sent in doc_sents:\n",
    "            if any ([ne[\"text\"] in ent_data[\"surface_forms\"] for ne in sent[\"nes\"]]):\n",
    "                sents_having.append(sent)\n",
    "            nes_belonging += [ne for ne in sent[\"nes\"] if ne[\"text\"] in ent_data[\"surface_forms\"]]\n",
    "        assert len(sents_having) > 0\n",
    "        assert len((nes_belonging)) >= len(ent_data[\"surface_forms\"])\n",
    "\n",
    "        doc_entities[longest][\"ne_cat\"] = Counter([ne[\"tag\"] for ne in nes_belonging]).most_common(1)[0][0]\n",
    "        doc_entities[longest][\"sentences_pol\"] = [s[\"sentence_pol\"] for s in sents_having]\n",
    "        doc_entities[longest][\"sent_pol_resolved\"] = resolve_sent_pols(doc_entities[longest][\"sentences_pol\"])\n",
    "        doc_entities[longest][\"targets_pol\"] = [t[\"tsa_pol\"] for t in nes_belonging]\n",
    "        doc_entities[longest][\"targ_pol_resolved\"] = resolve_sent_pols(doc_entities[longest][\"targets_pol\"])\n",
    "        doc_entities[longest][\"doc_rating\"] = doc_sents[0][\"doc_rating\"]\n",
    "\n",
    "\n",
    "    entity_dfs.append(pd.DataFrame(doc_entities).\n",
    "                                            T.\n",
    "                                            reset_index(level=0).\n",
    "                                            rename({\"index\":\"name\"}, axis=1)\n",
    "                                            )\n",
    "    entity_dict.update(doc_entities)\n",
    "\n",
    "\n",
    "merged_df = pd.concat(entity_dfs).reset_index()\n",
    "merged_df = merged_df [[\"doc_id\" , \"doc_rating\", \"entity_id\",  \"name\",\t\"surface_forms\", \"ne_cat\",\t\"sentences_pol\", \t\"sent_pol_resolved\", \"targets_pol\",\t\"targ_pol_resolved\"]]\n",
    "merged_df[\"manual_pol\"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_id</th>\n",
       "      <th>doc_rating</th>\n",
       "      <th>entity_id</th>\n",
       "      <th>name</th>\n",
       "      <th>surface_forms</th>\n",
       "      <th>ne_cat</th>\n",
       "      <th>sentences_pol</th>\n",
       "      <th>sent_pol_resolved</th>\n",
       "      <th>targets_pol</th>\n",
       "      <th>targ_pol_resolved</th>\n",
       "      <th>manual_pol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>105165</td>\n",
       "      <td>5</td>\n",
       "      <td>105165_Trine_Rein</td>\n",
       "      <td>Trine Rein</td>\n",
       "      <td>[Trine Rein, Trine]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Neutral, Neutral, Positive]</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[, , Positive]</td>\n",
       "      <td>Positive</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>105165</td>\n",
       "      <td>5</td>\n",
       "      <td>105165_Tony_Carey</td>\n",
       "      <td>Tony Carey</td>\n",
       "      <td>[Tony Carey]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Neutral]</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>105165</td>\n",
       "      <td>5</td>\n",
       "      <td>105165_Ritchie_Blackmore</td>\n",
       "      <td>Ritchie Blackmore</td>\n",
       "      <td>[Ritchie Blackmore]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Neutral]</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>105165</td>\n",
       "      <td>5</td>\n",
       "      <td>105165_Ronnie_James_Dios</td>\n",
       "      <td>Ronnie James Dios</td>\n",
       "      <td>[Ronnie James Dios]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Neutral]</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>105165</td>\n",
       "      <td>5</td>\n",
       "      <td>105165_Maria</td>\n",
       "      <td>Maria</td>\n",
       "      <td>[Maria]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Mixed]</td>\n",
       "      <td>Mixed</td>\n",
       "      <td>[Positive]</td>\n",
       "      <td>Positive</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>109227</td>\n",
       "      <td>5</td>\n",
       "      <td>109227_Leee_John</td>\n",
       "      <td>Leee John</td>\n",
       "      <td>[Leee John]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Neutral]</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>109227</td>\n",
       "      <td>5</td>\n",
       "      <td>109227_Imagination</td>\n",
       "      <td>Imagination</td>\n",
       "      <td>[Imagination]</td>\n",
       "      <td>ORG</td>\n",
       "      <td>[Neutral]</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>109227</td>\n",
       "      <td>5</td>\n",
       "      <td>109227_Axxe</td>\n",
       "      <td>Axxe</td>\n",
       "      <td>[Axxe]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Positive]</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>109227</td>\n",
       "      <td>5</td>\n",
       "      <td>109227_Tailors</td>\n",
       "      <td>Tailors</td>\n",
       "      <td>[Tailors]</td>\n",
       "      <td>ORG</td>\n",
       "      <td>[Positive]</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>109227</td>\n",
       "      <td>5</td>\n",
       "      <td>109227_Lars_Vaular</td>\n",
       "      <td>Lars Vaular</td>\n",
       "      <td>[Lars Vaular]</td>\n",
       "      <td>PER</td>\n",
       "      <td>[Positive]</td>\n",
       "      <td>Positive</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>293 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     doc_id doc_rating                 entity_id               name  \\\n",
       "0    105165          5         105165_Trine_Rein         Trine Rein   \n",
       "1    105165          5         105165_Tony_Carey         Tony Carey   \n",
       "2    105165          5  105165_Ritchie_Blackmore  Ritchie Blackmore   \n",
       "3    105165          5  105165_Ronnie_James_Dios  Ronnie James Dios   \n",
       "4    105165          5              105165_Maria              Maria   \n",
       "..      ...        ...                       ...                ...   \n",
       "288  109227          5          109227_Leee_John          Leee John   \n",
       "289  109227          5        109227_Imagination        Imagination   \n",
       "290  109227          5               109227_Axxe               Axxe   \n",
       "291  109227          5            109227_Tailors            Tailors   \n",
       "292  109227          5        109227_Lars_Vaular        Lars Vaular   \n",
       "\n",
       "           surface_forms ne_cat                 sentences_pol  \\\n",
       "0    [Trine Rein, Trine]    PER  [Neutral, Neutral, Positive]   \n",
       "1           [Tony Carey]    PER                     [Neutral]   \n",
       "2    [Ritchie Blackmore]    PER                     [Neutral]   \n",
       "3    [Ronnie James Dios]    PER                     [Neutral]   \n",
       "4                [Maria]    PER                       [Mixed]   \n",
       "..                   ...    ...                           ...   \n",
       "288          [Leee John]    PER                     [Neutral]   \n",
       "289        [Imagination]    ORG                     [Neutral]   \n",
       "290               [Axxe]    PER                    [Positive]   \n",
       "291            [Tailors]    ORG                    [Positive]   \n",
       "292        [Lars Vaular]    PER                    [Positive]   \n",
       "\n",
       "    sent_pol_resolved     targets_pol targ_pol_resolved manual_pol  \n",
       "0            Positive  [, , Positive]          Positive             \n",
       "1             Neutral              []                               \n",
       "2             Neutral              []                               \n",
       "3             Neutral              []                               \n",
       "4               Mixed      [Positive]          Positive             \n",
       "..                ...             ...               ...        ...  \n",
       "288           Neutral              []                               \n",
       "289           Neutral              []                               \n",
       "290          Positive              []                               \n",
       "291          Positive              []                               \n",
       "292          Positive              []                               \n",
       "\n",
       "[293 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the data\n",
    "basepath = os.path.join(tabular_savefolder, \"elsa_entities.\")\n",
    "merged_df.to_excel(basepath+\"xlsx\")\n",
    "merged_df.to_pickle(basepath+\"pk\")\n",
    "with open(basepath+\"json\", \"w\", encoding=\"utf-8\") as wf:\n",
    "    json.dump(entity_dict, wf, ensure_ascii=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we have extracted all information we can, and can hand this table together with the printouts to the annotator"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('transform')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "63f766e18cf02043d406c2f113693a415f1494f09983f05ef5cfd3ee3ed0acbc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}