LM-SD.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dtim-upc/THOR/blob/main/LM-SD.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hPt_dJlgQ94B"
      },
      "outputs": [],
      "source": [
        "# General Libraries\n",
        "import pandas as pd\n",
        "import srsly\n",
        "import json\n",
        "import os\n",
        "import re\n",
        "import csv\n",
        "import time\n",
        "from nervaluate import Evaluator\n",
        "import warnings\n",
        "import wandb\n",
        "\n",
        "# Spacy Related Imports\n",
        "import spacy\n",
        "from spacy.util import minibatch, compounding, compile_infix_regex, get_words_and_spaces\n",
        "from spacy.tokens import Span, DocBin, Doc\n",
        "from spacy.vocab import Vocab\n",
        "from spacy.tokenizer import Tokenizer\n",
        "from spacy.lang.en import English\n",
        "from spacy.scorer import Scorer\n",
        "from spacy.training import Example\n",
        "from print_dict import pd as pdic\n",
        "\n",
        "# RDFLib libraries\n",
        "from rdflib import Graph\n",
        "import pprint\n",
        "from rdflib import RDFS\n",
        "from rdflib import URIRef\n",
        "from rdflib.namespace import RDF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aALUh8d0T2MJ"
      },
      "outputs": [],
      "source": [
        "# wandb.login()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FsLNf7TUMeGm"
      },
      "source": [
        "# Data Conversion Part\n",
        "#### From Docanno Annotated Data for NER and RE into Spacy NER"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MamxT9nRnL0w"
      },
      "outputs": [],
      "source": [
        "TRAIN_DIR = 'dataset/sd_train'\n",
        "VALID_DIR = 'dataset/val'\n",
        "TEST_ON_TRAIN_GT_DIR = 'dataset/train'\n",
        "TEST_MAYO_DIR = 'dataset/test'\n",
        "CONFIG_DIR = 'config'\n",
        "OUTPUT_DIR = 'dataset'\n",
        "SCHEMA_FILE = \"dataset/schema/Disease_Schema_Extended.ttl\"\n",
        "STRUCTURED_DATA_DIR = \"dataset/csv\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QDMfN1PXT2MM"
      },
      "outputs": [],
      "source": [
        "def get_data_from_csv(file_name=\"\"):\n",
        "  '''\n",
        "  This function will get a CSV from the user having TWO COL i.e., Disease, Anatomy and\n",
        "  returns a Dictionary having a structure: {\"Tuberculosis\": ['lungs', 'brain', 'kidneys', 'spine'], ...}\n",
        "  '''\n",
        "\n",
        "  structured_data = {}\n",
        "\n",
        "  with open(file_name, 'r') as file:\n",
        "      csvreader = csv.reader(file)\n",
        "      # we can ignore the header\n",
        "      header = next(csvreader)\n",
        "      # print('Reading New Structured Data Source: {}'.format(file_name.split('\\\\')[1]))\n",
        "      # print('Data Headers: ', header)\n",
        "\n",
        "      for row in csvreader:\n",
        "        # # print(row)\n",
        "        # splits the comma separated values from the 1st column\n",
        "        first_cols = row[0].split(',')\n",
        "\n",
        "        # each of the instances of the first col (subject) will have the same value domain\n",
        "        for instance in first_cols:\n",
        "          instance = instance.strip()\n",
        "\n",
        "          # splits the comma separated values from the 2nd column\n",
        "          value_domain = row[1].split(',')\n",
        "          # removes the leading and trailing spaces\n",
        "          value_domain = [x.strip() for x in value_domain]\n",
        "          # making instance/value-domain dictionary\n",
        "          structured_data[instance] = value_domain\n",
        "\n",
        "  # print('Instances/Values:')\n",
        "  # print(structured_data)\n",
        "  # print()\n",
        "\n",
        "  return header, structured_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "myk9JvxYT2MN"
      },
      "outputs": [],
      "source": [
        "def sd_to_docanno(sd_data, is_reverse, S_P_O, TEXT_ID, ENT_ID, REL_ID, out_dir=TRAIN_DIR):\n",
        "    '''This function will convert the instance-value (e.g., 'Tuberculosis', 'lungs') pair of the structured data\n",
        "       into docanno formatted annotation for both NER and RE and save them into JSONL files\n",
        "    '''\n",
        "    # file_name will be the name of the relationship\n",
        "    file_name = out_dir + '/' + S_P_O[1] + '.jsonl'\n",
        "\n",
        "    # splitts the name of the relationship by removing the _ and R from it ('has_side_effect_R' -> ' has side effect ')\n",
        "    rel_name_spaced = ' '\n",
        "    for word in S_P_O[1].split('_')[0:-1]:\n",
        "        rel_name_spaced += word + ' '\n",
        "\n",
        "    with open(file_name, 'w') as json_file:\n",
        "        # if the relationship is inverse w.r.t how it is organized in the CSV\n",
        "        if is_reverse:\n",
        "            for instance, val_domains in sd_data.items():\n",
        "                for val in val_domains:\n",
        "                    text = val + rel_name_spaced + instance\n",
        "                    text_len = len(text)\n",
        "\n",
        "                    # Doccano formatted JSONL structure for NER and RE\n",
        "                    ent_1 = {'id':ENT_ID, \"label\": S_P_O[0], \"start_offset\": 0, \"end_offset\": len(val)}\n",
        "                    ent_2 = {'id':ENT_ID+1, \"label\": S_P_O[2], \"start_offset\": text_len - len(instance), \"end_offset\": text_len}\n",
        "                    rel = {\"id\": REL_ID, \"from_id\": ENT_ID, \"to_id\": ENT_ID+1, \"type\": S_P_O[1]}\n",
        "\n",
        "                    tem_docc_json = {\"id\": TEXT_ID, \"text\": text, \"entities\": [ent_1, ent_2], \"relations\": rel, \"Comments\": []}\n",
        "\n",
        "                    TEXT_ID += 1\n",
        "                    REL_ID +=1\n",
        "                    ENT_ID += 2\n",
        "\n",
        "                    # writing the annotation into JSONL file\n",
        "                    json_file.write(json.dumps(tem_docc_json))\n",
        "                    json_file.write('\\n')\n",
        "\n",
        "        else:\n",
        "            for instance, val_domains in sd_data.items():\n",
        "                for val in val_domains:\n",
        "                    text = instance + rel_name_spaced + val\n",
        "                    text_len = len(text)\n",
        "\n",
        "                    # Doccano formatted JSONL structure for NER and RE\n",
        "                    ent_1 = {'id':ENT_ID, \"label\": S_P_O[0], \"start_offset\": 0, \"end_offset\": len(instance)}\n",
        "                    ent_2 = {'id':ENT_ID+1, \"label\": S_P_O[2], \"start_offset\": text_len - len(val), \"end_offset\": text_len}\n",
        "                    rel = {\"id\": REL_ID, \"from_id\": ENT_ID, \"to_id\": ENT_ID+1, \"type\": S_P_O[1]}\n",
        "\n",
        "                    tem_docc_json = {\"id\": TEXT_ID, \"text\": text, \"entities\": [ent_1, ent_2], \"relations\": rel, \"Comments\": []}\n",
        "\n",
        "                    TEXT_ID += 1\n",
        "                    REL_ID +=1\n",
        "                    ENT_ID += 2\n",
        "\n",
        "                    # writing the annotation into JSONL file\n",
        "                    json_file.write(json.dumps(tem_docc_json))\n",
        "                    json_file.write('\\n')\n",
        "\n",
        "    return TEXT_ID, ENT_ID, REL_ID"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RoL0hg1BT2MO"
      },
      "outputs": [],
      "source": [
        "def structured_data_to_triples(templates, STRUCTURED_DATA_DIR=STRUCTURED_DATA_DIR):\n",
        "    '''This will read the Structured Data (SD) From the CSV Files of a Directory\n",
        "    '''\n",
        "\n",
        "    # id's for Annotating Docanno Formatted data from Structured Data\n",
        "    TEXT_ID = 0\n",
        "    ENT_ID = 0\n",
        "    REL_ID = 0\n",
        "\n",
        "    for dirname, _, filenames in os.walk(STRUCTURED_DATA_DIR):\n",
        "      for filename in filenames:\n",
        "        file_path = os.path.join(dirname, filename)\n",
        "        #print(file_path)\n",
        "        sd_header, sd_data = get_data_from_csv(file_path)\n",
        "        print(sd_header)\n",
        "\n",
        "        is_reverse = None\n",
        "        S_P_O = None\n",
        "\n",
        "        # matching this file with the templates to determine the type of relationship\n",
        "        for S, P, O in templates:\n",
        "            if sd_header[0] == S and sd_header[1] == O:\n",
        "                # there is an exact relationship\n",
        "                S_P_O = (S + '_E', P + '_R', O + '_E')\n",
        "                is_reverse = False\n",
        "                break\n",
        "            elif sd_header[0] == O and sd_header[1] == S:\n",
        "                # the relationship will be reversed\n",
        "                S_P_O = (S + '_E', P + '_R', O + '_E')\n",
        "                is_reverse = True\n",
        "                break\n",
        "\n",
        "        if is_reverse != None:\n",
        "            # calling the function to convert and save the SD into Docanno compatible annotation.\n",
        "            TEXT_ID, ENT_ID, REL_ID = sd_to_docanno(sd_data, is_reverse, S_P_O, TEXT_ID, ENT_ID, REL_ID, out_dir=TRAIN_DIR)\n",
        "        else:\n",
        "            print(f'Mismatch Between the Schema of the Graph and CSV Files in {sd_header}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JHhpWCohT2MO"
      },
      "outputs": [],
      "source": [
        "def get_rdf_graph(file_name=\"\"):\n",
        "      '''Getting the RDF file from the user (.nt/.ttl/.xml etc)'''\n",
        "\n",
        "      # if file_name is empty, that means we are using CoLAB Google Hosted Runtime\n",
        "      if not file_name:\n",
        "        file_name = input_file()\n",
        "\n",
        "      # parsing the graph\n",
        "      g = Graph()\n",
        "      g.parse(file_name)\n",
        "\n",
        "      # print('\\nTotal Triples Found = {}\\n'.format(len(g)))\n",
        "\n",
        "      # Loop through some of the triples in the graph (subj, pred, obj)\n",
        "#       print('First 10 Triples:')\n",
        "#       for triple in list(g)[:10]:\n",
        "#           # Check if there is at least one triple in the Graph\n",
        "#           # if (subj, pred, obj) not in g:\n",
        "#           #    raise Exception(\"It better be!\")\n",
        "#           print(triple)\n",
        "      print('RDF File Reading Complete...')\n",
        "\n",
        "      return g"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lFNwACKVT2MP"
      },
      "outputs": [],
      "source": [
        "def get_templates(g):\n",
        "  '''Given a RDF graph 'g' this function will return a list of (S, P, O) triples having only names'''\n",
        "\n",
        "  # Getting all the unique S, P, O from the graph using the RDFS.domain and RDFS.range\n",
        "  preds_subs = list(g.subject_objects(predicate=RDFS.domain))\n",
        "  preds_objs = list(g.subject_objects(predicate=RDFS.range))\n",
        "\n",
        "  # Getting all the subclasses with corresponding superclasses\n",
        "  sup_sub = g.subject_objects(predicate=RDFS.subClassOf)\n",
        "\n",
        "  # dictionary having a superclasses and it's subclasses {'Treatment':['Medicine', 'Precaution', 'Surgery']}\n",
        "  sup_sub_dic = {}\n",
        "\n",
        "  # populating the dictionary from the graph\n",
        "  for sc in sup_sub:\n",
        "    subj = sc[0].split('#')[1]\n",
        "    obj = sc[1].split('#')[1]\n",
        "\n",
        "    if obj in sup_sub_dic:\n",
        "      sup_sub_dic[obj].append(subj)\n",
        "    else:\n",
        "      sup_sub_dic[obj] = [subj]\n",
        "\n",
        "  # dictionary having a structure {'P':['S', 'O']}\n",
        "  dic_triples = {}\n",
        "\n",
        "  # gets only the name of Predicates and Subjects splitting from the URI's\n",
        "  for ps in preds_subs:\n",
        "    pred = ps[0].split('#')[1]\n",
        "    subj = ps[1].split('#')[1]\n",
        "    dic_triples[pred] = [subj]\n",
        "    # # print(subj, pred)\n",
        "\n",
        "  # matches the Subjects having specific Predicates with Objects\n",
        "  for po in preds_objs:\n",
        "    pred = po[0].split('#')[1]\n",
        "    obj = po[1].split('#')[1]\n",
        "    dic_triples[pred].append(obj)\n",
        "    # # print(pred, obj)\n",
        "\n",
        "  # saves the triples from the dictionary into a list of tuple -> [(S, P, O)]\n",
        "  triples_name = []\n",
        "  for pred in dic_triples:\n",
        "    subj = dic_triples[pred][0]\n",
        "    obj = dic_triples[pred][1]\n",
        "    triples_name.append((subj, pred, obj))\n",
        "    # # print('({}, {}, {})'.format(subj, pred, obj))\n",
        "\n",
        "    # checking if the subject is a superclass... If so, copy it's predicate to all it's subclasses (sub_cls) along with the range\n",
        "    if subj in sup_sub_dic:\n",
        "      for sub_cls in sup_sub_dic[subj]:\n",
        "        triples_name.append((sub_cls, pred, obj))\n",
        "\n",
        "  # print(\"Total Templates = {}\\n\".format(len(triples_name)))\n",
        "#   for triple in triples_name:\n",
        "#     print(triple)\n",
        "\n",
        "  return triples_name"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J9_0G1R3T2MR",
        "outputId": "09ec6279-bb6f-4e13-faec-4d43232f5b97"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RDF File Reading Complete...\n"
          ]
        }
      ],
      "source": [
        "# for local runtime - upload the file (first time) in the file upload option (left)\n",
        "graph = get_rdf_graph(file_name = SCHEMA_FILE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xw8jD3OIT2MS",
        "outputId": "f8f9d040-0cbb-4eb9-8c57-9808142af864"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[('Disease', 'affects', 'Anatomy'), ('Disease', 'caused_by', 'Cause'),\n",
            " ('Disease', 'has_code', 'Code'), ('Disease', 'has_complication', 'Complication'),\n",
            " ('Disease', 'has_diagnosis', 'Diagnosis'), ('Disease', 'has_precaution', 'Precaution'),\n",
            " ('Disease', 'has_risk_factor', 'Riskfactor'), ('Disease', 'has_symptom', 'Symptom'),\n",
            " ('Diagnosis', 'diagnosis_on', 'Anatomy'), ('Diagnosis', 'needs', 'Surgery'),\n",
            " ('Treatment', 'has_side_effect', 'Complication'),\n",
            " ('Medicine', 'has_side_effect', 'Complication'),\n",
            " ('Precaution', 'has_side_effect', 'Complication'),\n",
            " ('Surgery', 'has_side_effect', 'Complication'), ('Complication', 'influence', 'Anatomy'),\n",
            " ('Medicine', 'made_with', 'Composition'), ('Medicine', 'prescribed_for', 'Disease'),\n",
            " ('Surgery', 'surgery_for', 'Disease'), ('Surgery', 'surgery_on', 'Anatomy')]\n"
          ]
        }
      ],
      "source": [
        "templates = get_templates(graph)\n",
        "pdic(templates)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ISjLriFfT2MT",
        "outputId": "d047533e-83db-4201-da6d-bb6b2bf3ad04"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['Anatomy', 'Surgery']\n",
            "['Disease', 'Anatomy']\n",
            "['Disease', 'Cause']\n",
            "['Disease', 'Complication']\n",
            "['Diagnosis', 'Disease']\n",
            "['Disease', 'Medicine']\n",
            "['Disease', 'Precaution']\n",
            "['Disease', 'Riskfactor']\n",
            "['Disease', 'Symptom']\n",
            "['Medicine', 'Composition']\n"
          ]
        }
      ],
      "source": [
        "structured_data_to_triples(templates)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ff5LVLwARZpM",
        "outputId": "80b5c356-7096-4463-c095-055b2de4d83b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['Disease_E', 'Anatomy_E', 'Cause_E', 'Code_E', 'Diagnosis_E', 'Precaution_E', 'Riskfactor_E', 'Symptom_E', 'Medicine_E', 'Composition_E', 'Complication_E', 'Surgery_E']\n"
          ]
        }
      ],
      "source": [
        "\"\"\" This is just to SHOW the Named Entities - No Real Purpose \"\"\"\n",
        "ENTITY_LABELS = []\n",
        "for lbl in srsly.read_json(CONFIG_DIR + '/label_config_Entity.json'):\n",
        "  ENTITY_LABELS.append(lbl['text'])\n",
        "print(ENTITY_LABELS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8R23z1MypYhw"
      },
      "outputs": [],
      "source": [
        "def trim_entity_spans(text, spans):\n",
        "  '''Data Cleaning: Removes leading and trailing white spaces from entity spans.'''\n",
        "  invalid_span_tokens = re.compile(r'\\s')\n",
        "\n",
        "  valid_spans = []\n",
        "  for start, end, label in spans:\n",
        "    valid_start = start\n",
        "    valid_end = end\n",
        "    while valid_start < len(text) and invalid_span_tokens.match(text[valid_start]):\n",
        "      valid_start += 1\n",
        "    while valid_end > 1 and invalid_span_tokens.match(text[valid_end - 1]):\n",
        "      valid_end -= 1\n",
        "    valid_spans.append((valid_start, valid_end, label))\n",
        "\n",
        "  return valid_spans"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tKpws7x-Rq8j"
      },
      "outputs": [],
      "source": [
        "def docanno_to_spacy_ner_db(DATA_DIR):\n",
        "  '''\n",
        "  This function takes a directory of docanno annotated datasets for NER/RE\n",
        "  and converts them into SpaCy DocBin Object which is Trainable via commandline\n",
        "  '''\n",
        "  # Creates a blank Tokenizer with just the English vocab\n",
        "  nlp = spacy.blank(\"en\")\n",
        "\n",
        "  Doc.set_extension(\"rel\", default={},force=True)\n",
        "  vocab = Vocab()\n",
        "\n",
        "  word_count = 0\n",
        "  no_disease = 0\n",
        "  no_doc = 0\n",
        "  no_entities = 0\n",
        "  error_cnt = 0\n",
        "\n",
        "  # the DocBin will store the example documents\n",
        "  db = DocBin()\n",
        "\n",
        "  for dirname, _, filenames in os.walk(DATA_DIR):\n",
        "    for filename in filenames:\n",
        "      file_path = os.path.join(dirname, filename)\n",
        "\n",
        "      try:\n",
        "        \"\"\" Iterate through the Jsonl file to create serialize Docbin object / .spacy IOB File \"\"\"\n",
        "        for json_line in srsly.read_jsonl(file_path):\n",
        "\n",
        "          # parsing the docanno JSON data (per-line)\n",
        "          text = json_line[\"text\"]\n",
        "          spans = json_line[\"entities\"]\n",
        "\n",
        "          new_spans = []\n",
        "          for span in spans:\n",
        "            new_spans.append((span[\"start_offset\"], span[\"end_offset\"], span[\"label\"]))\n",
        "\n",
        "          # cleaning and validating the leading and trailing spaces from the annotated entities\n",
        "          valid_spans = trim_entity_spans(text, new_spans)\n",
        "\n",
        "          \"\"\" Parsing tokens from Text \"\"\"\n",
        "          tokens = nlp(text)\n",
        "\n",
        "          entities = []\n",
        "\n",
        "          spaces = [True if tok.whitespace_ else False for tok in tokens]\n",
        "          words = [t.text for t in tokens]\n",
        "          doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
        "\n",
        "          for start, end, label in valid_spans:\n",
        "            \"\"\" The modes should be: strict, contract, and expand \"\"\"\n",
        "            # print(eg['text'][int(span[\"start_offset\"]):int(span[\"end_offset\"])])\n",
        "            entity = doc.char_span(start, end, label=label, alignment_mode='contract')\n",
        "\n",
        "            # Not considering the spans which are Erroneous\n",
        "            if entity is None:\n",
        "              error_cnt += 1\n",
        "              #print(f'Error Found in File: {filename};\\n ID = {json_line[\"id\"]}; Label = {label}\\n')\n",
        "\n",
        "            else:\n",
        "              no_entities += 1\n",
        "              entities.append(entity)\n",
        "\n",
        "          # print(entities)\n",
        "          try:\n",
        "            doc.ents = entities\n",
        "            word_count += len(words)\n",
        "          except:\n",
        "            # print(\"=>> Error\")\n",
        "            continue\n",
        "\n",
        "          db.add(doc)\n",
        "          no_doc += 1\n",
        "\n",
        "      except:\n",
        "        print('Error While Loading JSON Data From Input Directory. Please check if you have other file type...')\n",
        "\n",
        "      no_disease +=1\n",
        "  print(f\"- Diseases: {no_disease} \\n- Processed Documents: {no_doc} \\n- Total Entities: {no_entities} \\n- Erroneous Entities (Ignored): {error_cnt} \\n- Total Words: {word_count}\")\n",
        "\n",
        "  return db"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y4NzFl50Wa9D",
        "outputId": "e92bb8f3-e779-45d4-f169-7820f72de4b6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Preparing Training Dataset from Structured Data:\n",
            "- Diseases: 10 \n",
            "- Processed Documents: 2353 \n",
            "- Total Entities: 4706 \n",
            "- Erroneous Entities (Ignored): 0 \n",
            "- Total Words: 14010\n",
            "\n",
            "Preparing Validation Dataset from Ground Truth Validation:\n",
            "- Diseases: 61 \n",
            "- Processed Documents: 366 \n",
            "- Total Entities: 3989 \n",
            "- Erroneous Entities (Ignored): 10 \n",
            "- Total Words: 41284\n",
            "\n",
            "Preparing Test Dataset from Ground Truth Training:\n",
            "- Diseases: 240 \n",
            "- Processed Documents: 1438 \n",
            "- Total Entities: 18539 \n",
            "- Erroneous Entities (Ignored): 34 \n",
            "- Total Words: 178882\n",
            "\n",
            "Preparing Test Dataset from Ground Truth Test:\n",
            "- Diseases: 1 \n",
            "- Processed Documents: 90 \n",
            "- Total Entities: 2222 \n",
            "- Erroneous Entities (Ignored): 6 \n",
            "- Total Words: 20588\n"
          ]
        }
      ],
      "source": [
        "'''Saving Spacy Trainable Object File for NER'''\n",
        "print('Preparing Training Dataset from Structured Data:')\n",
        "db_train = docanno_to_spacy_ner_db(TRAIN_DIR)\n",
        "db_train.to_disk(OUTPUT_DIR + \"/disease_A-Z_train.spacy\")\n",
        "\n",
        "print('\\nPreparing Validation Dataset from Ground Truth Validation:')\n",
        "db_valid = docanno_to_spacy_ner_db(VALID_DIR)\n",
        "db_valid.to_disk(OUTPUT_DIR + \"/disease_A-Z_valid.spacy\")\n",
        "\n",
        "print('\\nPreparing Test Dataset from Ground Truth Training:')\n",
        "db_test = docanno_to_spacy_ner_db(TEST_ON_TRAIN_GT_DIR)\n",
        "db_test.to_disk(OUTPUT_DIR + \"/disease_A-Z_test_on_train.spacy\")\n",
        "\n",
        "print('\\nPreparing Test Dataset from Ground Truth Test:')\n",
        "db_test = docanno_to_spacy_ner_db(TEST_MAYO_DIR)\n",
        "db_test.to_disk(OUTPUT_DIR + \"/disease_A-Z_test_mayo.spacy\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s4_72OGpMjs8"
      },
      "source": [
        "# Spacy Model Training Part"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xEbhYAYmMnte",
        "outputId": "0c07b8a3-8974-4dff-a1c1-1b5fe1b0c3d4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;2m[+] Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m[+] Saved config\u001b[0m\n",
            "config\\config.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ],
      "source": [
        "# Creates the training configuration file from the given base configuration. You can configure it yourself on:\n",
        "# https://spacy.io/usage/training#quickstart\n",
        "# We are using a GPU based training setting for Accuracy (RoBERTa model)\n",
        "\n",
        "#!python -m spacy init fill-config config/base_config.cfg config/config.cfg"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Yf1qRT1-P9W8",
        "outputId": "dd1015be-fcfc-4257-8354-1af24467c698"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[i] Saving to output directory: model\n",
            "[i] Using GPU: 0\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "[+] Initialized pipeline\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "[i] Pipeline: ['transformer', 'ner']\n",
            "[i] Initial learn rate: 0.0\n",
            "E    #       LOSS TRANS...  LOSS NER  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
            "---  ------  -------------  --------  ------  ------  ------  ------\n",
            "  0       0         167.39    301.06    2.38    1.32   12.66    0.02\n",
            " 10     200       26518.41  45397.28   14.32   12.39   16.95    0.14\n",
            " 21     400         154.33    133.78   10.85   11.91    9.95    0.11\n",
            " 31     600          54.45     48.30    8.28    9.42    7.40    0.08\n",
            " 42     800          11.82     11.48   10.06   10.17    9.95    0.10\n",
            " 52    1000          37.00     28.12    5.80    6.97    4.96    0.06\n",
            " 63    1200          52.99     45.81    9.84   10.82    9.02    0.10\n",
            " 73    1400          21.71     17.07    9.42   11.51    7.97    0.09\n",
            " 84    1600           0.00      0.00   10.54   12.21    9.28    0.11\n",
            " 94    1800          27.72     18.84    5.60    5.80    5.41    0.06\n",
            "105    2000          42.66     38.61    8.67    8.91    8.45    0.09\n",
            "115    2200         423.20    306.16    3.71    4.50    3.16    0.04\n",
            "126    2400           7.23      5.73    4.47    5.01    4.04    0.04\n",
            "136    2600         130.33    121.79    3.49    4.49    2.86    0.03\n",
            "147    2800        1713.16   1549.06    5.21    5.91    4.66    0.05\n",
            "157    3000          20.50     14.22    4.17    4.76    3.71    0.04\n",
            "168    3200          20.47     15.10    2.43    3.28    1.93    0.02\n",
            "178    3400           3.28      2.37    3.64    4.17    3.23    0.04\n",
            "189    3600          20.81     19.22    3.28    3.72    2.93    0.03\n",
            "200    3800          28.17     22.66    1.57    2.35    1.18    0.02\n",
            "210    4000          21.60     14.35    2.70    4.05    2.03    0.03\n",
            "221    4200          25.85     23.91    7.14    7.28    6.99    0.07\n",
            "231    4400           6.45      4.00    4.91    6.41    3.99    0.05\n",
            "242    4600          27.34     20.51    6.69    7.58    5.99    0.07\n",
            "252    4800           4.47      4.37    5.48    6.55    4.71    0.05\n",
            "263    5000          16.52      9.30    5.20    6.39    4.39    0.05\n",
            "273    5200          80.01     52.73    5.44    6.10    4.91    0.05\n",
            "[+] Saved pipeline to output directory\n",
            "model\\model-last\n",
            "Total Training Time = 3605.443698644638 (sec)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[2023-08-18 17:04:39,177] [INFO] Set up nlp object from config\n",
            "[2023-08-18 17:04:39,984] [INFO] Pipeline: ['transformer', 'ner']\n",
            "[2023-08-18 17:04:39,987] [INFO] Created vocabulary\n",
            "[2023-08-18 17:04:39,988] [INFO] Finished initializing nlp object\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModel: ['lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.dense.bias']\n",
            "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[2023-08-18 17:04:51,166] [INFO] Initialized pipeline components: ['transformer', 'ner']\n",
            "wandb: Currently logged in as: ataur (disease-team). Use `wandb login --relogin` to force relogin\n",
            "wandb: wandb version 0.15.8 is available!  To upgrade, please run:\n",
            "wandb:  $ pip install wandb --upgrade\n",
            "wandb: Tracking run with wandb version 0.15.5\n",
            "wandb: Run data is saved locally in E:\\DRIVE\\STUDY\\DEDS_PhD\\PhD\\Project\\#PhD_Proj\\LM_Structured_Data\\wandb\\run-20230818_170452-0l1z7eu1\n",
            "wandb: Run `wandb offline` to turn off syncing.\n",
            "wandb: Syncing run exalted-capybara-9\n",
            "wandb:  View project at https://wandb.ai/disease-team/THOR\n",
            "wandb:  View run at https://wandb.ai/disease-team/THOR/runs/0l1z7eu1\n",
            "wandb: Adding directory to artifact (.\\dataset)... Done. 0.3s\n",
            "wandb: Adding directory to artifact (.\\model\\model-last)... Done. 1.0s\n",
            "wandb: Adding directory to artifact (.\\model\\model-last)... Done. 0.9s\n",
            "wandb: Adding directory to artifact (.\\model\\model-last)... Done. 1.4s\n",
            "wandb: Adding directory to artifact (.\\model\\model-last)... Done. 1.1s\n",
            "wandb: Adding directory to artifact (.\\model\\model-last)... Done. 1.4s\n",
            "wandb: Waiting for W&B process to finish... (success).\n",
            "wandb: \n",
            "wandb: Run summary:\n",
            "wandb:           ents_f 0.05444\n",
            "wandb:           ents_p 0.06104\n",
            "wandb:           ents_r 0.04914\n",
            "wandb:         loss_ner 52.72546\n",
            "wandb: loss_transformer 80.00716\n",
            "wandb:            score 0.05444\n",
            "wandb:            speed 7286.09383\n",
            "wandb:        token_acc 1.0\n",
            "wandb:          token_f 1.0\n",
            "wandb:          token_p 1.0\n",
            "wandb:          token_r 1.0\n",
            "wandb: \n",
            "wandb:  View run exalted-capybara-9 at: https://wandb.ai/disease-team/THOR/runs/0l1z7eu1\n",
            "wandb: Synced 6 W&B file(s), 0 media file(s), 406 artifact file(s) and 0 other file(s)\n",
            "wandb: Find logs at: .\\wandb\\run-20230818_170452-0l1z7eu1\\logs\n"
          ]
        }
      ],
      "source": [
        "'''\n",
        "WARNING: Training Will Take Time -\n",
        "Trains a SpaCy NER model on our training data - Please REMOVE\n",
        "--gpu-id 0 if want to run this in CPU\n",
        "'''\n",
        "\n",
        "train_start_time = time.time()\n",
        "\n",
        "# !python -m spacy train config/config.cfg --gpu-id 0 --output model --paths.train dataset/disease_A-Z_train.spacy --paths.dev dataset/disease_A-Z_valid.spacy\n",
        "\n",
        "train_end_time = time.time()\n",
        "print(f'Total Training Time = {train_end_time - train_start_time} (sec)')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CB16GLeDvhzB"
      },
      "outputs": [],
      "source": [
        "# loading the best model from the directory (saved during the training)\n",
        "# Please download it from here: https://drive.google.com/file/d/1JlrIfJycQwQ3k9rLlOAIhYEWo9EIQExz/view?usp=sharing\n",
        "\n",
        "nlp_ner = spacy.load(\"LM-SD_Model/model-best\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "EtJiJSA3vpFh",
        "outputId": "b2419b7e-25c2-440c-b622-fc09e4aaade7"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Tuberculosis\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " generally damages \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    the lungs\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ", but it can also impair \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    other parts of\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " the body such as \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    brain and\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " spine\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    .\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " Typical signs of \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    active Tuberculosis\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " include \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    chronic cough\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " with \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    blood-containing mucus\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              ", fever, night sweats, and weight loss. \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Tuberculosis\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " damages the lungs whereas \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Malaria\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " could \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    detriment both\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              " kidneys by impairing the liver\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    .\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              "</div></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Inferencing and visualizing some sample text using the trained model\n",
        "text_inf = \"Tuberculosis generally damages the lungs, but it can also impair other parts of the body such as brain and spine. Typical signs of active Tuberculosis include chronic cough with blood-containing mucus, fever, night sweats, and weight loss. Tuberculosis damages the lungs whereas Malaria could detriment both kidneys by impairing the liver.\"\n",
        "\n",
        "doc_inf = nlp_ner(text_inf)\n",
        "\n",
        "colors = {'Disease_E': 'yellow', 'Anatomy_E': 'silver', 'Cause_E': '#0D9CB4',\n",
        "          'Code_E': '#5813C7', 'Diagnosis_E': '#0D350E', 'Precaution_E': '#1AA436',\n",
        "          'Riskfactor_E': '#1AE0F9', 'Symptom_E': 'orange', 'Medicine_E': '#BADCA1',\n",
        "          'Composition_E': '#78A2E5', 'Complication_E': '#D845FB', 'Surgery_E': '#54B69E'}\n",
        "options = {\"colors\": colors}\n",
        "\n",
        "spacy.displacy.render(doc_inf, style=\"ent\", options=options, jupyter=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "AukfFpwT6Bg7",
        "outputId": "da6caf5c-db35-4d2c-f680-4697729bc05c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Chagas (CHAH-gus)\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " disease is an \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    inflammatory, infectious disease\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              " caused by the \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    parasite Trypanosoma cruzi\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    . This parasite\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " is found in the \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    feces of the triatomine (reduviid) bug\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    . This bug\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " is also known as the 'kissing bug'. \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Chagas disease\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " is common in \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    South America\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ", Central America and Mexico, the primary home of the triatomine bug\n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    . Rare cases\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " of \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Chagas disease\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " have also been found in the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    southern United\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " States\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    .\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              "</div></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Inferencing and visualizing some sample text using the trained model\n",
        "text_inf = \"Chagas (CHAH-gus) disease is an inflammatory, infectious disease caused by the parasite Trypanosoma cruzi. This parasite is found in the feces of the triatomine (reduviid) bug. This bug is also known as the 'kissing bug'. Chagas disease is common in South America, Central America and Mexico, the primary home of the triatomine bug. Rare cases of Chagas disease have also been found in the southern United States.\"\n",
        "\n",
        "doc_inf = nlp_ner(text_inf)\n",
        "\n",
        "colors = {'Disease_E': 'yellow', 'Anatomy_E': 'silver', 'Cause_E': '#0D9CB4',\n",
        "          'Code_E': '#5813C7', 'Diagnosis_E': '#0D350E', 'Precaution_E': '#1AA436',\n",
        "          'Riskfactor_E': '#1AE0F9', 'Symptom_E': 'orange', 'Medicine_E': '#BADCA1',\n",
        "          'Composition_E': '#78A2E5', 'Complication_E': '#D845FB', 'Surgery_E': '#54B69E'}\n",
        "options = {\"colors\": colors}\n",
        "\n",
        "spacy.displacy.render(doc_inf, style=\"ent\", options=options, jupyter=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6MqiTgch_TWk",
        "outputId": "ac52ea33-7c96-4181-df68-54eeaf8b404c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[i] Using GPU: 0\n",
            "\u001b[1m\n",
            "================================== Results ==================================\u001b[0m\n",
            "\n",
            "TOK     100.00\n",
            "NER P   12.39 \n",
            "NER R   16.95 \n",
            "NER F   14.32 \n",
            "SPEED   3450  \n",
            "\n",
            "\u001b[1m\n",
            "=============================== NER (per type) ===============================\u001b[0m\n",
            "\n",
            "                     P       R       F\n",
            "Disease_E        18.89   42.86   26.22\n",
            "Complication_E    4.35    0.19    0.36\n",
            "Anatomy_E         3.89   10.78    5.72\n",
            "Cause_E           5.85   14.60    8.35\n",
            "Riskfactor_E      1.43    0.60    0.84\n",
            "Precaution_E     15.13   10.06   12.08\n",
            "Diagnosis_E       0.00    0.00    0.00\n",
            "Medicine_E       14.06   20.63   16.72\n",
            "Symptom_E         0.00    0.00    0.00\n",
            "Surgery_E         0.00    0.00    0.00\n",
            "Composition_E     0.00    0.00    0.00\n",
            "\n",
            "\n",
            "Total Validation Time = 19.92261505126953 (sec)\n"
          ]
        }
      ],
      "source": [
        "'''\n",
        "Strict Evaluation of the model separately - Using Ground Truth Validation Data\n",
        "This Evaluation Score does not consider Partial Match\n",
        "'''\n",
        "start_time = time.time()\n",
        "\n",
        "!python -m spacy evaluate --gpu-id 0 LM-SD_Model/model-best/ dataset/disease_A-Z_valid.spacy\n",
        "\n",
        "end_time = time.time()\n",
        "print(f'\\nTotal Validation Time = {end_time - start_time} (sec)')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hYS7kt-8T2MX",
        "outputId": "e667bd1a-cc45-4ecf-f80c-c75c8cd739a4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[i] Using GPU: 0\n",
            "\u001b[1m\n",
            "================================== Results ==================================\u001b[0m\n",
            "\n",
            "TOK     100.00\n",
            "NER P   13.24 \n",
            "NER R   17.09 \n",
            "NER F   14.92 \n",
            "SPEED   6619  \n",
            "\n",
            "\u001b[1m\n",
            "=============================== NER (per type) ===============================\u001b[0m\n",
            "\n",
            "                     P       R       F\n",
            "Disease_E        19.65   42.30   26.84\n",
            "Cause_E           7.08   13.89    9.38\n",
            "Symptom_E         4.20    0.26    0.50\n",
            "Complication_E    6.15    0.16    0.31\n",
            "Anatomy_E         4.80   11.75    6.81\n",
            "Medicine_E       13.96   20.22   16.52\n",
            "Precaution_E     18.23   17.54   17.87\n",
            "Surgery_E        26.32    1.08    2.07\n",
            "Composition_E     1.17   27.87    2.25\n",
            "Riskfactor_E      4.07    1.77    2.47\n",
            "Diagnosis_E       0.00    0.00    0.00\n",
            "\n",
            "\n",
            "Total Test Time = 36.59206771850586 (sec)\n"
          ]
        }
      ],
      "source": [
        "'''\n",
        "Strict Evaluation of the model separately - Using Ground Truth Training Data\n",
        "This Evaluation Score does not consider Partial Match\n",
        "'''\n",
        "start_time = time.time()\n",
        "\n",
        "!python -m spacy evaluate --gpu-id 0 LM-SD_Model/model-best/ dataset/disease_A-Z_test_on_train.spacy\n",
        "\n",
        "end_time = time.time()\n",
        "print(f'\\nTotal Test Time = {end_time - start_time} (sec)')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nmLGFJ7wK_o1",
        "outputId": "cb87386b-3964-4eac-f8ff-581aa2a77608"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[i] Using GPU: 0\n",
            "\u001b[1m\n",
            "================================== Results ==================================\u001b[0m\n",
            "\n",
            "TOK     100.00\n",
            "NER P   11.24 \n",
            "NER R   12.24 \n",
            "NER F   11.72 \n",
            "SPEED   2665  \n",
            "\n",
            "\u001b[1m\n",
            "=============================== NER (per type) ===============================\u001b[0m\n",
            "\n",
            "                     P       R       F\n",
            "Disease_E        15.65   40.73   22.61\n",
            "Anatomy_E         6.32    8.67    7.31\n",
            "Cause_E           3.72   23.40    6.41\n",
            "Complication_E   33.33    0.26    0.52\n",
            "Riskfactor_E      2.56    0.74    1.14\n",
            "Symptom_E         0.00    0.00    0.00\n",
            "Composition_E     1.50    4.62    2.26\n",
            "Medicine_E       29.10   10.37   15.29\n",
            "Precaution_E     10.43   23.61   14.47\n",
            "Diagnosis_E       0.00    0.00    0.00\n",
            "Surgery_E        20.00    1.18    2.22\n",
            "\n",
            "\n",
            "Total Test Time = 15.904410362243652 (sec)\n"
          ]
        }
      ],
      "source": [
        "'''\n",
        "Strict Evaluation of the model separately - Using Ground Truth Test Data\n",
        "This Evaluation Score does not consider Partial Match\n",
        "'''\n",
        "start_time = time.time()\n",
        "\n",
        "!python -m spacy evaluate --gpu-id 0 LM-SD_Model/model-best/ dataset/disease_A-Z_test_mayo.spacy\n",
        "\n",
        "end_time = time.time()\n",
        "print(f'\\nTotal Test Time = {end_time - start_time} (sec)')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "utAK07PIgTr5"
      },
      "source": [
        "# Evaluation using the Test/Validation Data [without cmd]\n",
        "## Need to Implement the Confusion Matrix (Sklearn)\n",
        "\n",
        "### Check the following Tutorial:\n",
        "https://github.com/wjbmattingly/spacy_tutorials_3x/blob/main/02_02_formal_test.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LqGJHtgPsgpY"
      },
      "outputs": [],
      "source": [
        "def load_data(file_path: str, nlp):\n",
        "  '''This function loads data from SpaCy docbin formatted files into spacy compitable JSON format'''\n",
        "  doc_bin = DocBin().from_disk(file_path)\n",
        "  samples, entities_count = [], 0\n",
        "  for doc in doc_bin.get_docs(nlp.vocab):\n",
        "    sample = {\n",
        "      \"text\": doc.text,\n",
        "      \"entities\": []\n",
        "    }\n",
        "    if len(doc.ents) > 0:\n",
        "      entities = [(e.start_char, e.end_char, e.label_) for e in doc.ents]\n",
        "      sample[\"entities\"] = entities\n",
        "      entities_count += len(entities)\n",
        "    else:\n",
        "      warnings.warn(\"Sample without entities!\")\n",
        "    samples.append(sample)\n",
        "  return samples, entities_count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HYmDFW0fsiRe"
      },
      "outputs": [],
      "source": [
        "samples_val, entities_count_val = load_data(OUTPUT_DIR + \"/disease_A-Z_valid.spacy\", nlp_ner)\n",
        "samples_train, entities_count_train = load_data(OUTPUT_DIR + \"/disease_A-Z_test_on_train.spacy\", nlp_ner)\n",
        "samples_test, entities_count_test = load_data(OUTPUT_DIR + \"/disease_A-Z_test_mayo.spacy\", nlp_ner)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hYSv0zQnt9Fu",
        "outputId": "ea19f89d-3073-4426-c9de-6913e6e140f0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'text': \"??????? - Abdominal aortic aneurysm - BD50.4Z\\nAneurysms can develop anywhere along the aorta, but most aortic aneurysms occur in the part of the aorta that's in the belly area (abdomen). Several things can play a role in the development of an abdominal aortic aneurysm, including: Hardening of the arteries (atherosclerosis). Atherosclerosis occurs when fat and other substances build up on the lining of a blood vessel. High blood pressure. High blood pressure can damage and weaken the aorta's walls. Blood vessel diseases. These are diseases that cause blood vessels to become inflamed. Infection in the aorta. Rarely, a bacterial or fungal infection might cause an abdominal aortic aneurysms. Trauma. For example, being injured in a car accident can cause an abdominal aortic aneurysms. \",\n",
              " 'entities': [(10, 35, 'Disease_E'),\n",
              "  (46, 55, 'Complication_E'),\n",
              "  (87, 92, 'Anatomy_E'),\n",
              "  (103, 119, 'Complication_E'),\n",
              "  (133, 150, 'Anatomy_E'),\n",
              "  (165, 185, 'Anatomy_E'),\n",
              "  (243, 268, 'Disease_E'),\n",
              "  (281, 324, 'Cause_E'),\n",
              "  (326, 341, 'Complication_E'),\n",
              "  (407, 419, 'Anatomy_E'),\n",
              "  (421, 440, 'Complication_E'),\n",
              "  (442, 461, 'Complication_E'),\n",
              "  (488, 501, 'Anatomy_E'),\n",
              "  (503, 524, 'Disease_E'),\n",
              "  (526, 544, 'Disease_E'),\n",
              "  (556, 588, 'Complication_E'),\n",
              "  (590, 612, 'Complication_E'),\n",
              "  (622, 653, 'Cause_E'),\n",
              "  (666, 695, 'Disease_E'),\n",
              "  (697, 703, 'Complication_E'),\n",
              "  (718, 749, 'Cause_E'),\n",
              "  (763, 789, 'Disease_E')]}"
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Looking into one particular samples from the ground truth of the validation set\n",
        "ground = samples_test[0]\n",
        "ground"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4SP-gt6B9t7k",
        "outputId": "02368e15-0077-4f7e-e09a-fddf2528ab3f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "??????? - Abdominal aortic aneurysm - BD50.4Z\n",
            "Aneurysms can develop anywhere along the aorta, but most aortic aneurysms occur in the part of the aorta that's in the belly area (abdomen). Several things can play a role in the development of an abdominal aortic aneurysm, including: Hardening of the arteries (atherosclerosis). Atherosclerosis occurs when fat and other substances build up on the lining of a blood vessel. High blood pressure. High blood pressure can damage and weaken the aorta's walls. Blood vessel diseases. These are diseases that cause blood vessels to become inflamed. Infection in the aorta. Rarely, a bacterial or fungal infection might cause an abdominal aortic aneurysms. Trauma. For example, being injured in a car accident can cause an abdominal aortic aneurysms. \n",
            "\n",
            "Phrase --> Predicted Entity\n",
            "\n",
            "?? --> Disease_E\n",
            "?? --> Disease_E\n",
            "?? --> Disease_E\n",
            "? - Abdominal aortic aneurysm --> Disease_E\n",
            "Aneurysms --> Disease_E\n",
            "the aorta --> Anatomy_E\n",
            "aortic aneurysms --> Disease_E\n",
            "belly area --> Anatomy_E\n",
            "abdominal aortic aneurysm --> Disease_E\n",
            "Hardening of the arteries --> Cause_E\n",
            "Atherosclerosis --> Disease_E\n",
            "fat and other substances --> Cause_E\n",
            "lining of a blood vessel --> Anatomy_E\n",
            "High blood pressure --> Disease_E\n",
            "High blood pressure --> Disease_E\n",
            "weaken the aorta's walls --> Anatomy_E\n",
            "Blood vessel diseases --> Anatomy_E\n",
            ". These are diseases --> Anatomy_E\n",
            "blood vessels to --> Anatomy_E\n",
            "Rarely, a bacterial or fungal infection --> Disease_E\n",
            "abdominal aortic aneurysms --> Anatomy_E\n",
            "Trauma --> Disease_E\n",
            ". For example --> Anatomy_E\n",
            ". --> Disease_E\n"
          ]
        }
      ],
      "source": [
        "# predicting the text of the above single sample with the model\n",
        "pred = nlp_ner(ground['text'])\n",
        "\n",
        "print(ground['text'])\n",
        "\n",
        "print('\\nPhrase --> Predicted Entity\\n')\n",
        "for ent in pred.ents:\n",
        "  print(ent.text, '-->', ent.label_)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t1Iq7LinFd8h"
      },
      "outputs": [],
      "source": [
        "def evaluate(ner_model, samples):\n",
        "  '''Given a trained spacy ner model along with json formatted data, this function will evaluate the model on the data'''\n",
        "  scorer = Scorer(ner_model)\n",
        "  example = []\n",
        "  for sample in samples:\n",
        "    pred = ner_model(sample['text'])\n",
        "    #print(type(pred))\n",
        "    temp_ex = Example.from_dict(pred, {'entities': sample['entities']})\n",
        "    example.append(temp_ex)\n",
        "  scores = scorer.score(example)\n",
        "  return scores"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jEX8TZHHJ7a5"
      },
      "outputs": [],
      "source": [
        "results = evaluate(nlp_ner, samples_val, )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L3SjYGpWKQt1",
        "outputId": "45d4cb82-07f4-4792-e4a0-71e68fa3b6bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{\n",
            "    'token_acc': 1.0,\n",
            "    'token_p': 1.0,\n",
            "    'token_r': 1.0,\n",
            "    'token_f': 1.0,\n",
            "    'ents_p': 0.12394572790612395,\n",
            "    'ents_r': 0.16946603158686388,\n",
            "    'ents_f': 0.1431748385047125,\n",
            "    'ents_per_type': {\n",
            "        'Disease_E': {\n",
            "            'p': 0.18885096700796358,\n",
            "            'r': 0.42857142857142855,\n",
            "            'f': 0.26217425638325875\n",
            "        },\n",
            "        'Complication_E': {\n",
            "            'p': 0.043478260869565216,\n",
            "            'r': 0.001876172607879925,\n",
            "            'f': 0.003597122302158273\n",
            "        },\n",
            "        'Anatomy_E': {\n",
            "            'p': 0.03891509433962264,\n",
            "            'r': 0.10784313725490197,\n",
            "            'f': 0.05719237435008665\n",
            "        },\n",
            "        'Cause_E': {\n",
            "            'p': 0.05849889624724062,\n",
            "            'r': 0.14600550964187328,\n",
            "            'f': 0.0835303388494878\n",
            "        },\n",
            "        'Riskfactor_E': {\n",
            "            'p': 0.014285714285714285,\n",
            "            'r': 0.005988023952095809,\n",
            "            'f': 0.008438818565400845\n",
            "        },\n",
            "        'Precaution_E': {\n",
            "            'p': 0.15126050420168066,\n",
            "            'r': 0.1005586592178771,\n",
            "            'f': 0.12080536912751676\n",
            "        },\n",
            "        'Diagnosis_E': {\n",
            "            'p': 0.0,\n",
            "            'r': 0.0,\n",
            "            'f': 0.0\n",
            "        },\n",
            "        'Medicine_E': {\n",
            "            'p': 0.140625,\n",
            "            'r': 0.20630372492836677,\n",
            "            'f': 0.16724738675958187\n",
            "        },\n",
            "        'Symptom_E': {\n",
            "            'p': 0.0,\n",
            "            'r': 0.0,\n",
            "            'f': 0.0\n",
            "        },\n",
            "        'Surgery_E': {\n",
            "            'p': 0.0,\n",
            "            'r': 0.0,\n",
            "            'f': 0.0\n",
            "        },\n",
            "        'Composition_E': {\n",
            "            'p': 0.0,\n",
            "            'r': 0.0,\n",
            "            'f': 0.0\n",
            "        }\n",
            "    }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "from print_dict import pd as pdic\n",
        "pdic(results)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0BjKlWOyT2MZ"
      },
      "source": [
        "# SemEval Evaluation Scripts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y7FpQLrYT2MZ"
      },
      "outputs": [],
      "source": [
        "def list_to_spacy_ner_doc(ner_pred):\n",
        "  '''\n",
        "  This function takes a list of directory of NER predictions of the form\n",
        "  {'text': '...', 'entities':[(start, end, tag)]} and converts them into SpaCy Doc Object\n",
        "  '''\n",
        "  # Creates a blank Tokenizer with just the English vocab\n",
        "  nlp = spacy.blank(\"en\")\n",
        "\n",
        "  Doc.set_extension(\"rel\", default={},force=True)\n",
        "  vocab = Vocab()\n",
        "\n",
        "  # try:\n",
        "  # parsing the docanno JSON data (per-line)\n",
        "  text = ner_pred[\"text\"]\n",
        "  spans = ner_pred[\"entities\"]\n",
        "\n",
        "  \"\"\" Parsing tokens from Text \"\"\"\n",
        "  tokens = nlp(text)\n",
        "\n",
        "  entities = []\n",
        "\n",
        "  spaces = [True if tok.whitespace_ else False for tok in tokens]\n",
        "  words = [t.text for t in tokens]\n",
        "  doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
        "\n",
        "  for start, end, label in spans:\n",
        "    \"\"\" The modes should be: strict, contract, and expand \"\"\"\n",
        "      # print(eg['text'][int(span[\"start_offset\"]):int(span[\"end_offset\"])])\n",
        "    entity = doc.char_span(start, end, label=label, alignment_mode='contract')\n",
        "\n",
        "    # Not considering the spans which are Erroneous\n",
        "    if entity is None:\n",
        "      # disease_name = text.split('\\n')[0]\n",
        "      # print(f'No Entity Found in File: {disease_name};\\n Span = {start}-{end}; Phrase = {doc.text[start:end]}; Label = {label}\\n')\n",
        "      continue\n",
        "    else:\n",
        "      entities.append(entity)\n",
        "\n",
        "  # print(entities[0].label_)\n",
        "  try:\n",
        "    doc.ents = entities\n",
        "  except:\n",
        "    print(\"=>> Error\")\n",
        "    print(text)\n",
        "\n",
        "  # except:\n",
        "  #   print('Error While Loading Predicted List...')\n",
        "\n",
        "  return doc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MHI3PMilT2Ma"
      },
      "outputs": [],
      "source": [
        "def render_sample_pred(ner_doc):\n",
        "  spacy.displacy.render(ner_doc, style=\"ent\", options=options, jupyter=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7Uc0Q3nUT2Ma",
        "outputId": "44246542-823a-4a06-8139-adddfe3606e6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'text': \"??????? - Abdominal aortic aneurysm - BD50.4Z\\nAneurysms can develop anywhere along the aorta, but most aortic aneurysms occur in the part of the aorta that's in the belly area (abdomen). Several things can play a role in the development of an abdominal aortic aneurysm, including: Hardening of the arteries (atherosclerosis). Atherosclerosis occurs when fat and other substances build up on the lining of a blood vessel. High blood pressure. High blood pressure can damage and weaken the aorta's walls. Blood vessel diseases. These are diseases that cause blood vessels to become inflamed. Infection in the aorta. Rarely, a bacterial or fungal infection might cause an abdominal aortic aneurysms. Trauma. For example, being injured in a car accident can cause an abdominal aortic aneurysms. \", 'entities': [(10, 35, 'Disease_E'), (46, 55, 'Complication_E'), (87, 92, 'Anatomy_E'), (103, 119, 'Complication_E'), (133, 150, 'Anatomy_E'), (165, 185, 'Anatomy_E'), (243, 268, 'Disease_E'), (281, 324, 'Cause_E'), (326, 341, 'Complication_E'), (407, 419, 'Anatomy_E'), (421, 440, 'Complication_E'), (442, 461, 'Complication_E'), (488, 501, 'Anatomy_E'), (503, 524, 'Disease_E'), (526, 544, 'Disease_E'), (556, 588, 'Complication_E'), (590, 612, 'Complication_E'), (622, 653, 'Cause_E'), (666, 695, 'Disease_E'), (697, 703, 'Complication_E'), (718, 749, 'Cause_E'), (763, 789, 'Disease_E')]}\n",
            "\n",
            "??????? - Abdominal aortic aneurysm - BD50.4Z\n",
            "Aneurysms can develop anywhere along the aorta, but most aortic aneurysms occur in the part of the aorta that's in the belly area (abdomen). Several things can play a role in the development of an abdominal aortic aneurysm, including: Hardening of the arteries (atherosclerosis). Atherosclerosis occurs when fat and other substances build up on the lining of a blood vessel. High blood pressure. High blood pressure can damage and weaken the aorta's walls. Blood vessel diseases. These are diseases that cause blood vessels to become inflamed. Infection in the aorta. Rarely, a bacterial or fungal infection might cause an abdominal aortic aneurysms. Trauma. For example, being injured in a car accident can cause an abdominal aortic aneurysms. \n",
            "\n",
            "Phrase --> Predicted Entity\n",
            "\n",
            "?? --> Disease_E\n",
            "?? --> Disease_E\n",
            "?? --> Disease_E\n",
            "? - Abdominal aortic aneurysm --> Disease_E\n",
            "Aneurysms --> Disease_E\n",
            "the aorta --> Anatomy_E\n",
            "aortic aneurysms --> Disease_E\n",
            "belly area --> Anatomy_E\n",
            "abdominal aortic aneurysm --> Disease_E\n",
            "Hardening of the arteries --> Cause_E\n",
            "Atherosclerosis --> Disease_E\n",
            "fat and other substances --> Cause_E\n",
            "lining of a blood vessel --> Anatomy_E\n",
            "High blood pressure --> Disease_E\n",
            "High blood pressure --> Disease_E\n",
            "weaken the aorta's walls --> Anatomy_E\n",
            "Blood vessel diseases --> Anatomy_E\n",
            ". These are diseases --> Anatomy_E\n",
            "blood vessels to --> Anatomy_E\n",
            "Rarely, a bacterial or fungal infection --> Disease_E\n",
            "abdominal aortic aneurysms --> Anatomy_E\n",
            "Trauma --> Disease_E\n",
            ". For example --> Anatomy_E\n",
            ". --> Disease_E\n"
          ]
        }
      ],
      "source": [
        "# Looking into one particular samples from the ground truth of the validation set\n",
        "ground = samples_test[0]\n",
        "print(ground)\n",
        "print()\n",
        "\n",
        "pred = nlp_ner(ground['text'])\n",
        "print(pred)\n",
        "\n",
        "print('\\nPhrase --> Predicted Entity\\n')\n",
        "for ent in pred.ents:\n",
        "  print(ent.text, '-->', ent.label_)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NhXuffOmT2Ma",
        "outputId": "0fa64310-0a3a-4c7c-c302-cdaf6312a90e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "########### Prediction ###########\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    ??\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    ??\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    ??\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    ? - Abdominal aortic aneurysm\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " - BD50.4Z</br>\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " can develop anywhere along \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    the aorta\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ", but most \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    aortic aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " occur in the part of the aorta that's in the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    belly area\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " (abdomen). Several things can play a role in the development of an \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    abdominal aortic aneurysm\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              ", including: \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Hardening of the arteries\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              " (atherosclerosis). \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Atherosclerosis\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " occurs when \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    fat and other substances\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              " build up on the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    lining of a blood vessel\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    High blood pressure\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    High blood pressure\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " can damage and \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    weaken the aorta's walls\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Blood vessel diseases\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    . These are diseases\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " that cause \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    blood vessels to\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " become inflamed. Infection in the aorta. \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Rarely, a bacterial or fungal infection\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " might cause an \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    abdominal aortic aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Trauma\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              "\n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    . For example\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ", being injured in a car accident can cause an abdominal aortic aneurysms\n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    .\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " </div></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "########### Ground Truth ###########\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">??????? - \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Abdominal aortic aneurysm\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " - BD50.4Z</br>\n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              " can develop anywhere along the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    aorta\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ", but most \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    aortic aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              " occur in the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    part of the aorta\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              " that's in the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    belly area (abdomen)\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ". Several things can play a role in the development of an \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    abdominal aortic aneurysm\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              ", including: \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Hardening of the arteries (atherosclerosis)\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Atherosclerosis\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              " occurs when fat and other substances build up on the lining of a \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    blood vessel\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    High blood pressure\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    High blood pressure\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              " can damage and weaken the \n",
              "<mark class=\"entity\" style=\"background: silver; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    aorta's walls\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Anatomy_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Blood vessel diseases\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    These are diseases\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              " that cause \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    blood vessels to become inflamed\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Infection in the aorta\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              ". Rarely, \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    a bacterial or fungal infection\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              " might cause \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    an abdominal aortic aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              ". \n",
              "<mark class=\"entity\" style=\"background: #D845FB; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Trauma\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Complication_E</span>\n",
              "</mark>\n",
              ". For example, \n",
              "<mark class=\"entity\" style=\"background: #0D9CB4; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    being injured in a car accident\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Cause_E</span>\n",
              "</mark>\n",
              " can cause an \n",
              "<mark class=\"entity\" style=\"background: yellow; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    abdominal aortic aneurysms\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Disease_E</span>\n",
              "</mark>\n",
              ". </div></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Visualizing the NER Predictions against the Ground Truth 'samples'\n",
        "print('\\n########### Prediction ###########\\n')\n",
        "render_sample_pred(pred)\n",
        "print('\\n########### Ground Truth ###########\\n')\n",
        "render_sample_pred(list_to_spacy_ner_doc(ground))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o31tZeDTT2Ma"
      },
      "outputs": [],
      "source": [
        "def spacy_evaluate(ner_predictions, samples, show_res=False):\n",
        "  '''Spacy Evaluation Function - Not going to use it\n",
        "  Given a trained spacy ner model along with json formatted data, this function will evaluate the model on the data'''\n",
        "  scorer = Scorer()\n",
        "  example = []\n",
        "  for pred, sample in zip(ner_predictions, samples):\n",
        "    # print(pred)\n",
        "    pred_doc = list_to_spacy_ner_doc(pred)\n",
        "\n",
        "    if show_res:\n",
        "      print('\\n########### Prediction ###########\\n')\n",
        "      render_sample_pred(pred_doc)\n",
        "      print('\\n########### Ground Truth ###########\\n')\n",
        "      render_sample_pred(list_to_spacy_ner_doc(sample))\n",
        "\n",
        "    temp_ex = Example.from_dict(pred_doc, {'entities': sample['entities']})\n",
        "    example.append(temp_ex)\n",
        "\n",
        "  scores = scorer.score(example)\n",
        "  return scores"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eqSBvmf-T2Mj"
      },
      "outputs": [],
      "source": [
        "def save_predictions(ner_predictions, filename, semeval_format=True, docanno_format=True):\n",
        "  # Saving the predictions as JSON - each dictionary on a line\n",
        "  semeval_ent = []\n",
        "  with open(OUTPUT_DIR+'/'+filename, 'w', encoding='UTF-8') as json_file:\n",
        "    for pred in ner_predictions:\n",
        "      tmp_ent = []\n",
        "      if semeval_format:\n",
        "        if docanno_format:\n",
        "            # Prodigy/Docanno formatted Ground Truth to work with nereval library - for SemEval 2013 - 9.1 task.\n",
        "            for ent in pred['entities']:\n",
        "              # saved in this format: [{\"label\": \"PER\", \"start\": 2, \"end\": 4}, ... ]\n",
        "              tmp_ent.append({\"label\": ent[2], \"start\": ent[0], \"end\": ent[1]})\n",
        "\n",
        "        else:\n",
        "            # Spacy Doc object to work with nereval library - for SemEval 2013 - 9.1 task.\n",
        "            for ent in pred.ents:\n",
        "              # saved in this format: [{\"label\": \"PER\", \"start\": 2, \"end\": 4}, ... ]\n",
        "              tmp_ent.append({\"label\": ent.label_, \"start\": ent.start_char, \"end\": ent.end_char})\n",
        "\n",
        "        semeval_ent.append(tmp_ent)\n",
        "\n",
        "      else:\n",
        "        # this is regullar spacy format, can be used for spacy's default evaluation later also for input of Ex-2\n",
        "        # saved in this format: {\"text\": \"\", \"entities\": [[36, 40, \"Complication_E\"], [44, 51, \"Anatomy_E\"], ...]}\n",
        "        for ent in pred.ents:\n",
        "            tmp_ent.append([ent.start_char, ent.end_char, ent.label_])\n",
        "\n",
        "        pred_json = {\"text\": pred.text, \"entities\": tmp_ent}\n",
        "        json_file.write(json.dumps(pred_json, ensure_ascii=False))\n",
        "        json_file.write('\\n')\n",
        "\n",
        "    if semeval_format:\n",
        "      # dumping it into a JSON file\n",
        "      json_file.write(json.dumps(semeval_ent, ensure_ascii=False))\n",
        "\n",
        "  return semeval_ent\n",
        "  # # This is single line JSON Dump of the entile list of dictionary - parser cannot parse it directly\n",
        "  # with open(OUTPUT_DIR+'/predition.jsonl', 'w') as fout:\n",
        "  #     json.dump(ner_predictions, fout)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iT3vYPLbT2Mk"
      },
      "outputs": [],
      "source": [
        "def preprocess_results(results_by_tag):\n",
        "    results_by_entity = []\n",
        "    for entity in ENTITY_LABELS:\n",
        "        if entity != 'Code_E':\n",
        "            df = pd.DataFrame(results_by_tag[entity])\n",
        "            df = df.round(decimals = 2)\n",
        "            df.insert(0,'Entity','')\n",
        "            df['Entity'] = entity\n",
        "            results_by_entity.append(df)\n",
        "    return results_by_entity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KkoZETpCT2Mk"
      },
      "outputs": [],
      "source": [
        "def semeval_evaluation(true, pred):\n",
        "    evaluator = Evaluator(true, pred, tags=ENTITY_LABELS)\n",
        "    results, results_by_tag = evaluator.evaluate()\n",
        "\n",
        "    results = pd.DataFrame(results)\n",
        "    results.to_excel(OUTPUT_DIR+'/'+'overall_benchmark.xlsx')\n",
        "\n",
        "    results_by_entity = pd.concat(preprocess_results(results_by_tag))\n",
        "    results_by_entity.to_excel(OUTPUT_DIR+'/'+'entity_benchmark.xlsx')\n",
        "\n",
        "    return results, results_by_entity"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "37Qm2ROgT2Mk"
      },
      "source": [
        "### Validation Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BQwQPQwrT2Ml"
      },
      "outputs": [],
      "source": [
        "start_time = time.time()\n",
        "# Saving the predictions in a list for Validation Set\n",
        "ner_predictions_val = []\n",
        "\n",
        "for sample in samples_val:\n",
        "    ner_predictions_val.append(nlp_ner(sample['text']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7PFWV7daT2Ml",
        "outputId": "bfc60e35-3278-4856-8344-cf9a75b60d87"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "#Causes - Quinsy - CA0K.1\n",
            "Peri Tonsillar Abscess is usually a complication of an untreated or partially treated acute tonsillitis. The infection, in these cases, spreads to the peritonsillar area (peritonsillitis). This region comprises loose connective tissue and is hence susceptible to formation of abscess.\n"
          ]
        }
      ],
      "source": [
        "print(ner_predictions_val[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yEefJ23xT2Ml"
      },
      "outputs": [],
      "source": [
        "# saving the ground and predictions into a JSONL file for later evaluation.\n",
        "semeval_ground_val = save_predictions(samples_val, filename= 'ground_val_semEval.jsonl')\n",
        "semeval_pred_val = save_predictions(ner_predictions_val, filename='predition_val_semEval.jsonl', semeval_format=True, docanno_format=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LDrP595_T2Ml",
        "outputId": "66cf9e2f-c1c3-4f30-b98e-683d77544832"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Total Validation Time = 239.03550362586975 (sec)\n"
          ]
        }
      ],
      "source": [
        "# Validation evaluation following SemEval 2013 metrics\n",
        "results, results_by_entity = semeval_evaluation(true=semeval_ground_val, pred=semeval_pred_val)\n",
        "end_time = time.time()\n",
        "print(f'\\nTotal Validation Time = {end_time - start_time} (sec)')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9nhh863_T2Mm"
      },
      "source": [
        "### Ground Truth Training Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zpn0nhCET2Mm"
      },
      "outputs": [],
      "source": [
        "start_time = time.time()\n",
        "# Saving the predictions in a list for Test Set\n",
        "ner_predictions_train = []\n",
        "\n",
        "for sample in samples_train:\n",
        "    ner_predictions_train.append(nlp_ner(sample['text']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5AvRX88OT2Mm"
      },
      "outputs": [],
      "source": [
        "# saving the grond and predictions into a JSONL file for later evaluation.\n",
        "semeval_ground_train = save_predictions(samples_train, filename= 'ground_train.jsonl')\n",
        "semeval_pred_train = save_predictions(ner_predictions_test, filename='predition_train.jsonl', semeval_format=True, docanno_format=False)\n",
        "\n",
        "# Saving this for Experiment 2... Spacy Format\n",
        "_ = save_predictions(ner_predictions_test, filename='predition_LM_spacy.jsonl', semeval_format=False, docanno_format=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "926o4orvT2Mw",
        "outputId": "a665b499-0d6f-4626-f3c4-9744a278fae2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Total Time for Prediction on Training Data = 303.4368019104004 (sec)\n"
          ]
        }
      ],
      "source": [
        "# Train evaluation following SemEval 2013 metrics\n",
        "results, results_by_entity = semeval_evaluation(true=semeval_ground_train, pred=semeval_pred_train)\n",
        "end_time = time.time()\n",
        "print(f'\\nTotal Time for Prediction on Training Data = {end_time - start_time} (sec)')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tVTM4KZjT2Mx"
      },
      "source": [
        "### Test Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sXzQTwCuT2Mx"
      },
      "outputs": [],
      "source": [
        "start_time = time.time()\n",
        "# Saving the predictions in a list for Test Set\n",
        "ner_predictions_test = []\n",
        "\n",
        "for sample in samples_test:\n",
        "    ner_predictions_test.append(nlp_ner(sample['text']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XEOGRF7ET2Mx"
      },
      "outputs": [],
      "source": [
        "# saving the grond and predictions into a JSONL file for later evaluation.\n",
        "semeval_ground_test = save_predictions(samples_test, filename= 'ground_test.jsonl')\n",
        "semeval_pred_test = save_predictions(ner_predictions_test, filename='predition_test.jsonl', semeval_format=True, docanno_format=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z0hdXdCgT2Mx",
        "outputId": "ab2e6399-1913-47d5-dae0-4a31e9929dc0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Total Test Time = 66.44146227836609 (sec)\n"
          ]
        }
      ],
      "source": [
        "# Test evaluation following SemEval 2013 metrics\n",
        "results, results_by_entity = semeval_evaluation(true=semeval_ground_test, pred=semeval_pred_test)\n",
        "end_time = time.time()\n",
        "print(f'\\nTotal Test Time = {end_time - start_time} (sec)')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rgX4Q96XT2My",
        "outputId": "9be2d161-0ef8-45d8-a4b4-051d5719256d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "########### Overall Results ###########\n",
            "\n",
            "Precision: 0.4173895084675754\n",
            "Recall: 0.4547704770477048\n",
            "F1: 0.43527891449493866\n",
            "\n"
          ]
        }
      ],
      "source": [
        "    print('\\n########### Overall Results ###########\\n')\n",
        "    print(f\"Precision: {results['partial']['precision']}\\nRecall: {results['partial']['recall']}\\nF1: {results['partial']['f1']}\\n\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.16"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}