POS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Current setup using this notebook\n",
    "<p>label - POS</p>\n",
    "<p>Fine tune on: raw-train</p>\n",
    "<p>Evaluate on: Raw-dev (and Raw-test, but not reported)</p>\n",
    "<p>Classification by: whole word (as opposed to prefix/host)</p>\n",
    "<p>Morphologically informed labels? None </p>\n",
    "<p>Shuffle/Sort? shuffle </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "import bclm\n",
    "\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from transformers import BertTokenizer, BertConfig\n",
    "from transformers import BertForTokenClassification, AdamW"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manually setting seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(3)\n",
    "np.random.seed(3)\n",
    "torch.cuda.manual_seed_all(3)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data\n",
    "`bclm.read_dataframe('spmrl', subset='train')` - gives the gold-segmented tokens <br>\n",
    "`bclm.get_token_df(train, ['upostag'])` - gives the raw tokens<br>\n",
    "`bclm.read_dataframe('yap_dev')` - gives the YAP tokenization (only available on `yap_dev` and `yap_test`. No `yap_train`!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = bclm.read_dataframe('spmrl', subset='train')\n",
    "train_df = bclm.get_token_df(train, ['upostag', 'biose'])\n",
    "train_df['token_str'] = train_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "dev = bclm.read_dataframe('spmrl', subset='dev')\n",
    "dev_df = bclm.get_token_df(dev, ['upostag', 'biose'])\n",
    "dev_df['token_str'] = dev_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "test = bclm.read_dataframe('spmrl', subset='test')\n",
    "test_df = bclm.get_token_df(test, ['upostag', 'biose'])\n",
    "test_df['token_str'] = test_df['token_str'].str.replace('”','\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dev_df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uniform column names\n",
    "Note that the column names in yap dfs can be slightly different from spmrl dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Evaluating on Raw-dev/Raw-test\n",
    "train_df.rename(columns = {\"token_str\": \"form\"}, inplace = True)\n",
    "dev_df.rename(columns = {\"token_str\": \"form\"}, inplace = True)\n",
    "test_df.rename(columns = {\"token_str\": \"form\"}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## For Evaluating on Yap-dev/Yap-test\n",
    "# dev_df.rename(columns = {\"misc_token_id\": \"token_id\"}, inplace = True)\n",
    "# test_df.rename(columns = {\"misc_token_id\": \"token_id\"}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dev_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### Add sorting on the dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df['sent_len'] = train_df.groupby('sent_id').id.transform('size')\n",
    "# s = train_df.sort_values(by=['sent_len', 'sent_id', 'id']).index\n",
    "# train_df_sorted = train_df.reindex(s)\n",
    "# train_df_sorted.groupby('sent_id', sort=False).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train_df_sorted.iloc[150:200]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get lists of sentences\n",
    "Note that longest sentences from dev and test splits need to be removed, and those sentences change depending on wether or not sorting was applied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class sentenceGetter(object):\n",
    "    def __init__(self, data, max_sent=None):\n",
    "        self.index = 0\n",
    "        self.max_sent = max_sent\n",
    "        self.tokens = data['form']\n",
    "        self.labels = data['upostag']\n",
    "        #for evaluating by word-accuracy\n",
    "        self.correspondingToken = data['token_id']\n",
    "        self.orig_sent_id = data['sent_id']\n",
    "    \n",
    "    def sentences(self):\n",
    "        sent = []\n",
    "        counter = 0\n",
    "        \n",
    "        for token,label, corres_tok, sent_id in zip(self.tokens, self.labels, self.correspondingToken, self.orig_sent_id):\n",
    "            sent.append((token, label, corres_tok, sent_id))\n",
    "            if token.strip() == \".\":\n",
    "                yield sent\n",
    "                sent = []\n",
    "                counter += 1\n",
    "            if self.max_sent is not None and counter >= self.max_sent:\n",
    "                return\n",
    "\n",
    "train_getter = sentenceGetter(train_df)\n",
    "dev_getter = sentenceGetter(dev_df)\n",
    "test_getter = sentenceGetter(test_df)\n",
    "\n",
    "train_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "train_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "\n",
    "dev_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "\n",
    "test_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "\n",
    "print(train_sentences[10])\n",
    "print(train_labels[10])\n",
    "\n",
    "print(len(dev_sentences))\n",
    "print(len(test_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Get the longest sentences in the dev and test sets\n",
    "longest_sent_len = 0\n",
    "for sent in dev_sentences:\n",
    "    if len(sent) >= longest_sent_len:\n",
    "        print(len(sent))\n",
    "        longest_sent_len = len(sent)\n",
    "        print(\"index of longest sentence:{} \".format(dev_sentences.index(sent)))\n",
    "        \n",
    "longest_sent_len = 0\n",
    "for sent in test_sentences:\n",
    "    if len(sent) >= longest_sent_len:\n",
    "        print(len(sent))\n",
    "        longest_sent_len = len(sent)\n",
    "        print(\"index of longest sentence:{} \".format(test_sentences.index(sent)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Remove too long sentences\n",
    "\n",
    "del dev_sentences[296]\n",
    "del dev_labels[296]\n",
    "del dev_corresTokens[296]\n",
    "del dev_sent_ids[296]\n",
    "\n",
    "del dev_sentences[226]\n",
    "del dev_labels[226]\n",
    "del dev_corresTokens[226]\n",
    "del dev_sent_ids[226]\n",
    "\n",
    "del dev_sentences[57]\n",
    "del dev_labels[57]\n",
    "del dev_corresTokens[57]\n",
    "del dev_sent_ids[57]\n",
    "\n",
    "del dev_sentences[49]\n",
    "del dev_labels[49]\n",
    "del dev_corresTokens[49]\n",
    "del dev_sent_ids[49]\n",
    "\n",
    "\n",
    "del test_sentences[396]\n",
    "del test_labels[396]\n",
    "del test_corresTokens[396]\n",
    "del test_sent_ids[396]\n",
    "\n",
    "del test_sentences[164]\n",
    "del test_labels[164]\n",
    "del test_corresTokens[164]\n",
    "del test_sent_ids[164]\n",
    "\n",
    "del test_sentences[157]\n",
    "del test_labels[157]\n",
    "del test_corresTokens[157]\n",
    "del test_sent_ids[157]\n",
    "\n",
    "del test_sentences[151]\n",
    "del test_labels[151]\n",
    "del test_corresTokens[151]\n",
    "del test_sent_ids[151]\n",
    "\n",
    "# ## YAP deletions\n",
    "# del dev_sentences[296]\n",
    "# del dev_labels[296]\n",
    "# del dev_corresTokens[296]\n",
    "# del dev_sent_ids[296]\n",
    "\n",
    "# del dev_sentences[226]\n",
    "# del dev_labels[226]\n",
    "# del dev_corresTokens[226]\n",
    "# del dev_sent_ids[226]\n",
    "\n",
    "# del dev_sentences[57]\n",
    "# del dev_labels[57]\n",
    "# del dev_corresTokens[57]\n",
    "# del dev_sent_ids[57]\n",
    "\n",
    "# del dev_sentences[49]\n",
    "# del dev_labels[49]\n",
    "# del dev_corresTokens[49]\n",
    "# del dev_sent_ids[49]\n",
    "\n",
    "# del dev_sentences[24]\n",
    "# del dev_labels[24]\n",
    "# del dev_corresTokens[24]\n",
    "# del dev_sent_ids[24]\n",
    "\n",
    "# del dev_sentences[22]\n",
    "# del dev_labels[22]\n",
    "# del dev_corresTokens[22]\n",
    "# del dev_sent_ids[22]\n",
    "\n",
    "# del dev_sentences[12]\n",
    "# del dev_labels[12]\n",
    "# del dev_corresTokens[12]\n",
    "# del dev_sent_ids[12]\n",
    "\n",
    "# del dev_sentences[9]\n",
    "# del dev_labels[9]\n",
    "# del dev_corresTokens[9]\n",
    "# del dev_sent_ids[9]\n",
    "\n",
    "# del dev_sentences[5]\n",
    "# del dev_labels[5]\n",
    "# del dev_corresTokens[5]\n",
    "# del dev_sent_ids[5]\n",
    "\n",
    "# del test_sentences[386]\n",
    "# del test_labels[386]\n",
    "# del test_corresTokens[386]\n",
    "# del test_sent_ids[386]\n",
    "\n",
    "# del test_sentences[384]\n",
    "# del test_labels[384]\n",
    "# del test_corresTokens[384]\n",
    "# del test_sent_ids[384]\n",
    "\n",
    "# del test_sentences[377]\n",
    "# del test_labels[377]\n",
    "# del test_corresTokens[377]\n",
    "# del test_sent_ids[377]\n",
    "\n",
    "# del test_sentences[213]\n",
    "# del test_labels[213]\n",
    "# del test_corresTokens[213]\n",
    "# del test_sent_ids[213]\n",
    "\n",
    "# del test_sentences[141]\n",
    "# del test_labels[141]\n",
    "# del test_corresTokens[141]\n",
    "# del test_sent_ids[141]\n",
    "\n",
    "# del test_sentences[124]\n",
    "# del test_labels[124]\n",
    "# del test_corresTokens[124]\n",
    "# del test_sent_ids[124]\n",
    "\n",
    "# del test_sentences[45]\n",
    "# del test_labels[45]\n",
    "# del test_corresTokens[45]\n",
    "# del test_sent_ids[45]\n",
    "\n",
    "# del test_sentences[35]\n",
    "# del test_labels[35]\n",
    "# del test_corresTokens[35]\n",
    "# del test_sent_ids[35]\n",
    "\n",
    "# del test_sentences[23]\n",
    "# del test_labels[23]\n",
    "# del test_corresTokens[23]\n",
    "# del test_sent_ids[23]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.set_device(0)\n",
    "\n",
    "print(\"Device: \" + str(device))\n",
    "print(\"Number of gpus: \" + str(n_gpu))\n",
    "print(\"Name of gpu: \" + torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 150\n",
    "bs = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('/home/nlp/aharonr6/git/AlephBert/models/alephbert-256-new-wiki-16k/checkpoint-205000', do_lower_case=False)\n",
    "def tokenize(sentences, orig_labels):\n",
    "    tokenized_texts = []\n",
    "    labels = []\n",
    "    for sent, sent_labels in zip(sentences, orig_labels):\n",
    "        bert_tokens = []\n",
    "        bert_labels = []\n",
    "        for orig_token, orig_label in zip(sent, sent_labels):\n",
    "            b_tokens = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(b_tokens)\n",
    "            for b_token in b_tokens:\n",
    "                bert_labels.append(orig_label)\n",
    "        tokenized_texts.append(bert_tokens)\n",
    "        labels.append(bert_labels)\n",
    "        assert len(bert_tokens) == len(bert_labels)\n",
    "    return tokenized_texts, labels\n",
    "\n",
    "train_tokenized_texts, train_tokenized_labels = tokenize(train_sentences, train_labels)\n",
    "print(train_tokenized_texts[10])\n",
    "print(train_tokenized_labels[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = train_df\n",
    "tag_vals = list(set(data[\"upostag\"].values))\n",
    "tags = ['PAD'] + tag_vals\n",
    "tag2idx = {tag:idx for idx, tag in enumerate(tags)}\n",
    "idx2tag = {idx:tag for idx, tag in enumerate(tags)}\n",
    "\n",
    "print(tag2idx)\n",
    "# print(idx2tag)\n",
    "print(len(tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentences_and_labels(tokenized_texts, labels):\n",
    "    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                              maxlen = MAX_LEN, dtype = \"float32\", truncating = \"post\", padding = \"post\", value = tag2idx['PAD'])\n",
    "    tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels], \n",
    "                         maxlen = MAX_LEN, value = tag2idx['PAD'], padding = \"post\",\n",
    "                        dtype = \"float32\", truncating = \"post\")\n",
    "    attention_masks = [[float(i>0) for i in ii] for ii in input_ids]\n",
    "    return input_ids, tags, attention_masks\n",
    "\n",
    "input_ids, tags, attention_masks = pad_sentences_and_labels(train_tokenized_texts, train_tokenized_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "tr_tags = torch.tensor(tags, dtype=torch.long)\n",
    "tr_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from transformers import get_linear_schedule_with_warmup\n",
    "\n",
    "model = BertForTokenClassification.from_pretrained('/home/nlp/aharonr6/git/AlephBert/models/alephbert-256-new-wiki-16k/checkpoint-205000',\n",
    "                                                   num_labels=len(tag2idx),\n",
    "                                                   output_attentions = False,\n",
    "                                                   output_hidden_states = False)\n",
    "model.cuda()\n",
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters())\n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "\n",
    "optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5, eps=1e-8)\n",
    "\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "#     print (pred_flat, labels_flat)\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "epochs = 15\n",
    "max_grad_norm = 1.0\n",
    "\n",
    "# Total number of training steps is number of batches * number of epochs.\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps=0,\n",
    "                                            num_training_steps=total_steps)\n",
    "\n",
    "## Store the average loss after each epoch so we can plot them.\n",
    "loss_values, validation_loss_values = [], []\n",
    "for _ in trange(epochs, desc=\"Epoch\"):\n",
    "    # TRAIN loop\n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        model.zero_grad()\n",
    "        # forward pass\n",
    "        outputs = model(b_input_ids, token_type_ids=None,\n",
    "                     attention_mask=b_input_mask, labels=b_labels)\n",
    "        # get the loss\n",
    "        loss = outputs[0]\n",
    "        # Perform a backward pass to calculate the gradients.\n",
    "        loss.backward()\n",
    "        # track train loss\n",
    "        total_loss += loss.item() \n",
    "        # Clip the norm of the gradient\n",
    "        # This is to help prevent the \"exploding gradients\" problem.\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        # Update the learning rate.\n",
    "        scheduler.step()\n",
    "        \n",
    "    # Calculate the average loss over the training data.\n",
    "    avg_train_loss = total_loss / len(train_dataloader)\n",
    "    print(\"Average train loss: {}\".format(avg_train_loss))\n",
    "    \n",
    "    # Store the loss value for plotting the learning curve.\n",
    "    loss_values.append(avg_train_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function receives a sentence with its labels, and the tokenized sentence and labels\n",
    "def aggr_toks_labels_tags(orig_words, orig_labels, tok_wordps, tok_labels, predicted_tags):\n",
    "    \n",
    "    joint_tokens = []\n",
    "    joint_labels = []\n",
    "    joint_predicted = []\n",
    "#     joint_test = []\n",
    "    \n",
    "    for word in orig_words:\n",
    "        aggregated_tokenized = \"\"\n",
    "        aggregated_label = \"\"\n",
    "        aggregated_predicted = \"\"\n",
    "        aggregated_test = \"\"\n",
    "        \n",
    "        while aggregated_tokenized != word:\n",
    "#             print(len(tok_sent))\n",
    "            tmpTok = tok_wordps.pop(0)\n",
    "#             print(tmpTok)\n",
    "#             print(joint_tokens)\n",
    "            if tmpTok.startswith(\"##\"):\n",
    "                tmpTok = tmpTok[2:]\n",
    "                \n",
    "            tmpLab = tok_labels.pop(0)\n",
    "#             if aggregated_label == \"\":\n",
    "            aggregated_label += '^'\n",
    "            aggregated_label += tmpLab\n",
    "\n",
    "                \n",
    "            tmpPred = predicted_tags.pop(0)\n",
    "#             print(tmpPred)\n",
    "\n",
    "            aggregated_predicted += '^'\n",
    "            aggregated_predicted += tmpPred\n",
    "#             if aggregated_predicted == \"\":\n",
    "#                 aggregated_predicted = tmpPred\n",
    "                \n",
    "#             tmpTest = test_tags.pop(0)\n",
    "#             if aggregated_test == \"\":\n",
    "#                 aggregated_test = tmpTest\n",
    "                \n",
    "            aggregated_tokenized += tmpTok\n",
    "#             print(aggregated_tokenized)\n",
    "            \n",
    "        joint_tokens.append(aggregated_tokenized)\n",
    "        joint_labels.append(aggregated_label)\n",
    "        joint_predicted.append(aggregated_predicted)\n",
    "#         joint_test.append(aggregated_test)\n",
    "        \n",
    "    assert len(joint_tokens) == len(orig_words)\n",
    "    assert len(joint_tokens) == len(joint_predicted)\n",
    "    return joint_tokens, joint_labels, joint_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "def delete_pads_from_preds(predicted_tags, test_tags):\n",
    "    clean_predicted = []\n",
    "    clean_test = []\n",
    "    \n",
    "    for ix in range(0, len(test_tags)):\n",
    "        if test_tags[ix] != 'PAD':\n",
    "            clean_predicted.append(predicted_tags[ix])\n",
    "            clean_test.append(test_tags[ix])\n",
    "            \n",
    "    return clean_predicted, clean_test\n",
    "    \n",
    "def calculate_accuracy(df):\n",
    "    numOfCorrectPredictions = 0\n",
    "    for index in df.index:\n",
    "        orig_pos = df.at[index, 'orig_label']\n",
    "        pred_pos = df.at[index, 'predicted_tag']\n",
    "        if orig_pos == pred_pos:\n",
    "            numOfCorrectPredictions += 1\n",
    "    return numOfCorrectPredictions/len(df)\n",
    "                \n",
    "def test_model(sentence, labels, tok_sent, tok_labels, corres_tokens, sent_id):\n",
    "    input_ids, tags, attention_masks = pad_sentences_and_labels([tok_sent], [tok_labels])\n",
    "\n",
    "    val_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "    val_tags = torch.tensor(tags, dtype=torch.long)\n",
    "    val_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "    test_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "    test_sampler = SequentialSampler(test_data)\n",
    "    test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=bs)\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions, true_labels = [], []\n",
    "    counter = 0\n",
    "    for batch in test_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = model(b_input_ids, token_type_ids=None,\n",
    "                                attention_mask=b_input_mask, labels=b_labels)\n",
    "        logits = outputs[1].detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        predictions.append([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        \n",
    "        true_labels.append(label_ids)\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "\n",
    "        eval_loss += outputs[0].mean().item()\n",
    "        eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "\n",
    "        nb_eval_examples += b_input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    \n",
    "    pred_tags = [idx2tag[p_ii] for p in predictions for p_i in p for p_ii in p_i]\n",
    "    joint_tokenized, joint_labels, preds = aggr_toks_labels_tags(sentence, labels, tok_sent, tok_labels, \n",
    "                                                                        pred_tags)\n",
    "    \n",
    "    tmp = {'word': sentence, 'orig_label': labels, 'predicted_tag': preds, \n",
    "           'corresToken': corres_tokens, 'sent_id': sent_id}\n",
    "    tmp_df = pd.DataFrame(data=tmp)\n",
    "    return tmp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_dev_df = pd.DataFrame()\n",
    "dev_tokenized_texts, dev_tokenized_labels = tokenize(dev_sentences, dev_labels)\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(dev_sentences, dev_labels, dev_tokenized_texts, \n",
    "                                                                   dev_tokenized_labels, dev_corresTokens, \n",
    "                                                                   dev_sent_ids):\n",
    "    eval_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_dev_df = full_dev_df.append(eval_df, ignore_index=True, sort=False)\n",
    "\n",
    "# full_df\n",
    "f1_accuracy = calculate_accuracy(full_dev_df)\n",
    "print(\"Accuracy (F1): = {}\".format(f1_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_test_df = pd.DataFrame()\n",
    "test_tokenized_texts, test_tokenized_labels = tokenize(test_sentences, test_labels)\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(test_sentences, test_labels, test_tokenized_texts, \n",
    "                                                                   test_tokenized_labels, test_corresTokens, \n",
    "                                                                   test_sent_ids):\n",
    "    eval_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_test_df = full_test_df.append(eval_df, ignore_index=True, sort=False)\n",
    "\n",
    "# full_df\n",
    "f1_accuracy = calculate_accuracy(full_test_df)\n",
    "print(\"Accuracy (F1): = {}\".format(f1_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_dev_df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # For evaluating on gold-dev/gold-test only - regrouping the tokens to words\n",
    "\n",
    "# dev_predicted = full_dev_df.groupby(['sent_id', 'corresToken']).apply(lambda x: '^'.join(x.predicted_tag)).reset_index()\n",
    "# dev_original = full_dev_df.groupby(['sent_id', 'corresToken']).apply(lambda x: '^'.join(x.orig_label)).reset_index()\n",
    "# dev_combined = pd.merge(dev_original, dev_predicted, on=['sent_id', 'corresToken'])\n",
    "# dev_combined.rename(columns = {\"0_x\": \"orig_label\", \"0_y\":\"predicted_tag\"}, inplace = True)\n",
    "\n",
    "# test_predicted = full_test_df.groupby(['sent_id', 'corresToken']).apply(lambda x: '^'.join(x.predicted_tag)).reset_index()\n",
    "# test_original = full_test_df.groupby(['sent_id', 'corresToken']).apply(lambda x: '^'.join(x.orig_label)).reset_index()\n",
    "# test_combined = pd.merge(test_original, test_predicted, on=['sent_id', 'corresToken'])\n",
    "# test_combined.rename(columns = {\"0_x\": \"orig_label\", \"0_y\":\"predicted_tag\"}, inplace = True)\n",
    "# # word_acc_test = full_test_df.groupby(['sent_id', 'corresToken']).apply(lambda x: '^'.join(x.predicted_tag)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev_combined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev_combined.to_csv('ftRaw_evalGoldDev_whole_pos_shuffle.csv')\n",
    "# test_combined.to_csv('ftRaw_evalGoldTest_whole_pos_shuffle.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from more_itertools import unique_everseen\n",
    "\n",
    "def unique_vals_to_list(df):\n",
    "    for index in df.index:\n",
    "        joint_pred = df.at[index, 'predicted_tag']\n",
    "        joint_orig = df.at[index, 'orig_label']\n",
    "        \n",
    "        predicted_tag_list = joint_pred.split('^')\n",
    "        predicted_tag_list_no_empty = list(filter(None, predicted_tag_list))\n",
    "        original_tag_list = joint_orig.split('^')\n",
    "        original_tag_list_no_empty = list(filter(None, original_tag_list))\n",
    "\n",
    "        \n",
    "        df.at[index, 'predicted_tag'] = list(unique_everseen(predicted_tag_list_no_empty))\n",
    "        df.at[index, 'orig_label'] = list(unique_everseen(original_tag_list_no_empty))\n",
    "        \n",
    "        \n",
    "unique_vals_to_list(full_dev_df)\n",
    "unique_vals_to_list(full_test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dev_combined.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exact_match_accuracy(df):\n",
    "    exact_matches = 0\n",
    "    for index in df.index:\n",
    "        if df.at[index, 'orig_label'] == df.at[index, 'predicted_tag']:\n",
    "            exact_matches += 1\n",
    "            \n",
    "    return exact_matches\n",
    "\n",
    "print(\"DEV - Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_dev_df)/len(full_dev_df) * 100))\n",
    "print(\"TEST - Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_test_df)/len(full_test_df) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def existence_accuracy(df):\n",
    "    # correct tag = appeared in predicted and in gold\n",
    "    total_orig_num_of_labels = 0\n",
    "    total_predicted_num_of_labels = 0\n",
    "    total_num_of_correct_tags = 0\n",
    "    \n",
    "    for index in df.index:\n",
    "        orig_list = df.at[index, 'orig_label']\n",
    "        predicted_list = df.at[index, 'predicted_tag']\n",
    "        total_orig_num_of_labels += len(orig_list)\n",
    "        total_predicted_num_of_labels += len(predicted_list)\n",
    "        total_num_of_correct_tags += len(set(orig_list).intersection(set(predicted_list)))\n",
    "        \n",
    "    precision = total_num_of_correct_tags / total_predicted_num_of_labels * 100\n",
    "    recall = total_num_of_correct_tags / total_orig_num_of_labels * 100\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "    print(\"Precision: {0:.2f}%\".format(precision))\n",
    "    print(\"Recall: {0:.2f}%\".format(recall))\n",
    "    print(\"F1: {0:.2f}%\".format(f1))\n",
    "    \n",
    "print(\"DEV:\")\n",
    "existence_accuracy(full_dev_df)\n",
    "print(\"TEST:\")\n",
    "existence_accuracy(full_test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_dev_df.iloc[30:55]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dev_df[dev_df['upostag'] == 'DEF^NN'])/len(dev_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "morphbert",
   "language": "python",
   "name": "morphbert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}