7-BERT-NER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT\n",
    "\n",
    "Dans ce notebook on utilise les classes de PyTorch `TransformerEncoderLayer`, `TransformerEncoder` pour implémenter un modèle BERT pour la reconnaissance d'entités nommées.\n",
    "\n",
    "## Préparer les données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install torchtext==0.6.0\n",
    "\n",
    "import time\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.optim import Adam\n",
    "from torchtext.data import Field, NestedField, BucketIterator\n",
    "from torchtext.datasets import SequenceTaggingDataset\n",
    "from torchtext.vocab import Vocab\n",
    "from collections import Counter\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "\n",
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import time\n",
    "import random\n",
    "import string\n",
    "from itertools import chain\n",
    "\n",
    "import numpy as np\n",
    "import copy\n",
    "from copy import deepcopy\n",
    "import torch.optim as optim\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "TEXT = data.Field(lower = True) \n",
    "TAG = data.Field(unk_token = None) \n",
    "CHAR_NESTING= Field(tokenize=list)\n",
    "CHAR = NestedField(CHAR_NESTING) \n",
    "\n",
    "train_data, valid_data, test_data = data.TabularDataset.splits(\n",
    "        path=\"data_ner/\",\n",
    "        train=\"train.csv\",\n",
    "        validation=\"valid.csv\",\n",
    "        test=\"test.csv\", format='csv', skip_header=True,\n",
    "        fields=(\n",
    "            ((\"text\", \"char\"), (TEXT, CHAR)), \n",
    "            (\"tag\", TAG)\n",
    "        )\n",
    "    )\n",
    "\n",
    "MIN_FREQ = 2\n",
    "\n",
    "TEXT.build_vocab(train_data, \n",
    "                 min_freq = MIN_FREQ, # les mots qui apparaissent moins que MIN_FREQ fois seront ignorés du vocabulaire\n",
    "                 vectors = \"glove.6B.300d\",\n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "\n",
    "TAG.build_vocab(train_data)\n",
    "CHAR.build_vocab(train_data) \n",
    "BATCH_SIZE = 32\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE,\n",
    "    device = device, sort=False)\n",
    "\n",
    "# padding index\n",
    "TEXT_PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "CHAR_PAD_IDX = CHAR.vocab.stoi[CHAR.pad_token]  \n",
    "TAG_PAD_IDX = TAG.vocab.stoi[TAG.pad_token]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construire le modèle\n",
    "\n",
    "Le modèle BERT est décrit dans cet article et est composé de N = 12 transformers composés de 6 encoders et d'une couche linéaire.\n",
    "\n",
    "Nous utilisons les paramètres du modèle BERT de l'article $BERT_{BASE}$ avec : \n",
    " \n",
    " - N = 12 transformers \n",
    " - Dimension des états cachés : fc_hidden = 768 \n",
    " - Têtes d'attention : attn_heads = 12\n",
    " \n",
    "La fonction `clones` suivante permet de produire N couches identiques et sera utile pour assembler N=12 modèles transformers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clones(module, N):\n",
    "    \"Produit N couches identiques\"\n",
    "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On implémente le même modèle transformer que dans un précédent notebook dans une classe à part."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, dropout=0.1, max_len=500):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)\n",
    "\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, embedding_dim, char_emb_dim, char_cnn_filter_num, attn_heads, dropout, trf_layers,fc_hidden, output_dim):\n",
    "        super(Transformer, self).__init__()\n",
    "        \n",
    "        all_emb_size = embedding_dim + (char_emb_dim * char_cnn_filter_num) \n",
    "        self.position_encoder = PositionalEncoding(\n",
    "            d_model=all_emb_size\n",
    "        )\n",
    "        \n",
    "        encoder_layers = nn.TransformerEncoderLayer(\n",
    "            d_model=all_emb_size,\n",
    "            nhead=attn_heads,\n",
    "            activation=\"relu\",\n",
    "            dropout=dropout\n",
    "        )\n",
    "        self.encoder = nn.TransformerEncoder(\n",
    "            encoder_layer=encoder_layers,\n",
    "            num_layers=trf_layers\n",
    "        )\n",
    " \n",
    "                \n",
    "        self.fc1 = nn.Linear(\n",
    "        in_features=all_emb_size,\n",
    "        out_features=fc_hidden\n",
    "        )\n",
    "        self.fc1_gelu = nn.GELU()\n",
    "        self.fc1_norm = nn.LayerNorm(fc_hidden)\n",
    "        self.fc2_dropout = nn.Dropout(dropout)\n",
    "        self.fc2 = nn.Linear(\n",
    "            in_features=fc_hidden,\n",
    "            out_features=output_dim\n",
    "        ) \n",
    "    def forward(self, words, word_features, word_pad_idx):\n",
    "        \n",
    "        key_padding_mask = torch.as_tensor(words == word_pad_idx).permute(1, 0)\n",
    "        # pos_out = [sentence length, batch size, embedding dim + char emb dim * num filter]\n",
    "        pos_out = self.position_encoder(word_features)\n",
    "        # enc_out = [sentence length, batch size, embedding dim + char emb dim * num filter]\n",
    "        enc_out = self.encoder(pos_out, src_key_padding_mask=key_padding_mask)\n",
    "        # Fully-connected\n",
    "        # fc1_out = [sentence length, batch size, fc hidden]\n",
    "        fc1_out = self.fc1_norm(self.fc1_gelu(self.fc1(enc_out)))\n",
    "        # fc2_out = [sentence length, batch size, output dim]\n",
    "        fc2_out = self.fc2(self.fc2_dropout(fc1_out))\n",
    "        return fc2_out\n",
    "\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "La classe BERT contient alors l'embedding des mots et des caractères et l'assemblage des N transformers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class BERT(nn.Module):\n",
    "\n",
    "    def __init__(self,\n",
    "                 input_dim,\n",
    "                 embedding_dim,\n",
    "                 char_emb_dim,\n",
    "                 char_input_dim,\n",
    "                 char_cnn_filter_num,\n",
    "                 char_cnn_kernel_size,\n",
    "                 attn_heads,\n",
    "                 fc_hidden,\n",
    "                 trf_layers,\n",
    "                 output_dim,\n",
    "                 dropout,\n",
    "                 word_pad_idx,\n",
    "                 char_pad_idx,\n",
    "                 tag_pad_idx,\n",
    "                 N):  \n",
    "        super().__init__()\n",
    "        self.N = N\n",
    "        self.char_pad_idx = char_pad_idx\n",
    "        self.word_pad_idx = word_pad_idx\n",
    "        self.tag_pad_idx = tag_pad_idx\n",
    "        # LAYER 1A: Word Embedding\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.embedding = nn.Embedding(\n",
    "            num_embeddings=input_dim,\n",
    "            embedding_dim=embedding_dim,\n",
    "            padding_idx=word_pad_idx\n",
    "        )\n",
    "        self.emb_dropout = nn.Dropout(dropout)\n",
    "        # LAYER 1B: Char Embedding-CNN\n",
    "        self.char_emb_dim = char_emb_dim\n",
    "        self.char_emb = nn.Embedding(\n",
    "            num_embeddings=char_input_dim,\n",
    "            embedding_dim=char_emb_dim,\n",
    "            padding_idx=char_pad_idx\n",
    "        )\n",
    "        self.char_cnn = nn.Conv1d(\n",
    "            in_channels=char_emb_dim,\n",
    "            out_channels=char_emb_dim * char_cnn_filter_num,\n",
    "            kernel_size=char_cnn_kernel_size,\n",
    "            groups=char_emb_dim  # different 1d conv for each embedding dim\n",
    "        )\n",
    "        self.cnn_dropout = nn.Dropout(dropout)\n",
    "        self.layers = clones(Transformer(embedding_dim, char_emb_dim, char_cnn_filter_num, attn_heads, dropout, trf_layers,fc_hidden, output_dim), N)\n",
    "        #self.trans = Transformer(embedding_dim, char_emb_dim, char_cnn_filter_num, attn_heads, trf_dropout, trf_layers,fc_hidden, output_dim)\n",
    "\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "\n",
    "\n",
    "        \n",
    "          \n",
    "        \n",
    "        \n",
    "        # init weights from normal distribution\n",
    "        for name, param in self.named_parameters():\n",
    "            nn.init.normal_(param.data, mean=0, std=0.1)\n",
    "\n",
    "    def forward(self, words, chars, tags=None):\n",
    "        # words = [sentence length, batch size]\n",
    "        # chars = [batch size, sentence length, word length)\n",
    "        # tags = [sentence length, batch size]\n",
    "        # embedding_out = [sentence length, batch size, embedding dim]\n",
    "        embedding_out = self.emb_dropout(self.embedding(words))\n",
    "        # character cnn layer forward\n",
    "        # reference: https://github.com/achernodub/targer/blob/master/src/layers/layer_char_cnn.py\n",
    "        # char_emb_out = [batch size, sentence length, word length, char emb dim]\n",
    "        char_emb_out = self.emb_dropout(self.char_emb(chars))\n",
    "        batch_size, sent_len, word_len, char_emb_dim = char_emb_out.shape\n",
    "        char_cnn_max_out = torch.zeros(batch_size, sent_len, self.char_cnn.out_channels) \n",
    "        for sent_i in range(sent_len):\n",
    "            # sent_char_emb = [batch size, word length, char emb dim]\n",
    "            sent_char_emb = char_emb_out[:, sent_i, :, :]\n",
    "            # sent_char_emb_p = [batch size, char emb dim, word length]\n",
    "            sent_char_emb_p = sent_char_emb.permute(0, 2, 1)\n",
    "            # char_cnn_sent_out = [batch size, out channels * char emb dim, word length - kernel size + 1]\n",
    "            char_cnn_sent_out = self.char_cnn(sent_char_emb_p)\n",
    "            char_cnn_max_out[:, sent_i, :], _ = torch.max(char_cnn_sent_out, dim=2)\n",
    "        char_cnn = self.cnn_dropout(char_cnn_max_out)\n",
    "        # concat word and char embedding\n",
    "        # char_cnn_p = [sentence length, batch size, char emb dim * num filter]\n",
    "        char_cnn_p = char_cnn.permute(1, 0, 2).to(device)\n",
    "        word_features = torch.cat((embedding_out, char_cnn_p), dim=2)\n",
    "        \n",
    "        \n",
    "        for i in range(self.N):\n",
    "            x = self.layers[i](words, word_features, self.word_pad_idx)    \n",
    "        return self.softmax(x)\n",
    "\n",
    "    def init_embeddings(self, char_pad_idx=CHAR_PAD_IDX, word_pad_idx=TEXT_PAD_IDX,pretrained=None, freeze=True):\n",
    "        # initialize embedding for padding as zero\n",
    "        self.embedding.weight.data[self.word_pad_idx] = torch.zeros(self.embedding_dim)\n",
    "        self.char_emb.weight.data[self.char_pad_idx] = torch.zeros(self.char_emb_dim)\n",
    "        if pretrained is not None:\n",
    "            self.embedding = nn.Embedding.from_pretrained(\n",
    "                embeddings=torch.as_tensor(pretrained),\n",
    "                padding_idx=self.word_pad_idx,\n",
    "                freeze=freeze\n",
    "            )\n",
    "    def count_parameters(self):\n",
    "        return sum(p.numel() for p in self.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(nombre transformers=12,  Hidden size=768,  att_head=12,  Total  Param-eters=110M)  \n",
    "\n",
    "`embedding_dim` + `char_emb_dim` * `char_cnn_filter_num` = `emb dim total`\n",
    "e use a batch size of 32 and fine-tune for 3epochs over the data for all GLUE tasks. \n",
    "For eachtask, we selected the best fine-tuning learning rate(among 5e-5, 4e-5, 3e-5, and 2e-5) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le modèle a 39,276,774 paramètres à entraîner.\n"
     ]
    }
   ],
   "source": [
    "model = BERT(\n",
    "    input_dim=len(TEXT.vocab),\n",
    "    embedding_dim=300,\n",
    "    char_emb_dim=45,  \n",
    "    char_input_dim=len(CHAR.vocab),\n",
    "    char_cnn_filter_num=4, \n",
    "    char_cnn_kernel_size=3,\n",
    "    attn_heads=12,  \n",
    "    fc_hidden=768,  \n",
    "    trf_layers=1,\n",
    "    output_dim=len(TAG.vocab),\n",
    "    dropout=0.7,\n",
    "    word_pad_idx=TEXT_PAD_IDX,\n",
    "    char_pad_idx=CHAR_PAD_IDX,\n",
    "    tag_pad_idx=TAG_PAD_IDX,\n",
    "    N = 12\n",
    ")\n",
    "model.init_embeddings(\n",
    "    char_pad_idx=CHAR_PAD_IDX,\n",
    "    word_pad_idx=TEXT_PAD_IDX,\n",
    "    pretrained= TEXT.vocab.vectors,\n",
    "    freeze=True\n",
    ")\n",
    "\n",
    "print(f\"Le modèle a {model.count_parameters():,} paramètres à entraîner.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr = 5e-5)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy_per_tag(predictions, tags):\n",
    "    n_tags = len(TAG.vocab)\n",
    "    class_correct = list(0 for i in range(n_tags))\n",
    "    class_total = list(0 for i in range(n_tags))\n",
    "    acc = list(0 for i in range(n_tags))\n",
    "    _, pred = torch.max(predictions, 1)\n",
    "    # # compare predictions to true label\n",
    "    correct = np.squeeze(pred.eq(tags.data.view_as(pred)))\n",
    "    # # calculate test accuracy for each object class\n",
    "    for i in range(len(tags.data)):\n",
    "        label = tags.data[i]\n",
    "        class_correct[label] += correct[i].item()\n",
    "        class_total[label] += 1\n",
    "    for i in range(n_tags):\n",
    "        if np.sum(class_total[i]) == 0 and np.sum(class_correct[i]) ==0:\n",
    "            res = 100\n",
    "        else:\n",
    "            res = 100 * class_correct[i] / class_total[i]\n",
    "        acc[i] = res, np.sum(class_correct[i]), np.sum(class_total[i])\n",
    "        \n",
    "    return acc  \n",
    "\n",
    "from sklearn.metrics import f1_score\n",
    "def f1_loss(preds, y, tag_pad_idx):\n",
    "    index_o = TAG.vocab.stoi[\"O\"]\n",
    "    positive_labels = [i for i in range(len(TAG.vocab.itos))\n",
    "                           if i not in (tag_pad_idx, index_o)]\n",
    "    _, pred = torch.max(preds, 1)\n",
    "    pred = pred.data.cpu().numpy() \n",
    "    tags = y.data.cpu().numpy()\n",
    "    f1 = f1_score(\n",
    "            y_true=tags,\n",
    "            y_pred=pred,\n",
    "            labels=positive_labels,\n",
    "            average=\"micro\"\n",
    "        ) \n",
    "       \n",
    "    return f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 2m 29s\n",
      "\tTrain Loss: 0.941 | Train F1 score: 0.15%\n",
      "\t Val. Loss: 0.994 |  Val. F1 score: 0.00%\n",
      "Epoch: 03 | Epoch Time: 2m 32s\n",
      "\tTrain Loss: 0.813 | Train F1 score: 0.00%\n",
      "\t Val. Loss: 0.912 |  Val. F1 score: 0.00%\n",
      "Epoch: 05 | Epoch Time: 2m 28s\n",
      "\tTrain Loss: 0.798 | Train F1 score: 0.00%\n",
      "\t Val. Loss: 0.881 |  Val. F1 score: 0.00%\n"
     ]
    }
   ],
   "source": [
    "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        text = batch.text\n",
    "        tags = batch.tag\n",
    "        chars = batch.char \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        predictions = model(text,chars)\n",
    "        \n",
    "        #predictions = [sent len, batch size, output dim]\n",
    "        #tags = [sent len, batch size]\n",
    "        predictions = predictions.view(-1, predictions.shape[-1])\n",
    "        tags = tags.view(-1)\n",
    "        \n",
    "        #predictions = [sent len * batch size, output dim]\n",
    "        #tags = [sent len * batch size]\n",
    "        \n",
    "        loss = criterion(predictions, tags)\n",
    "                \n",
    "        f1 = f1_loss(predictions, tags, tag_pad_idx)\n",
    "        acc = accuracy_per_tag(predictions, tags)   \n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), acc, epoch_f1 / len(iterator)\n",
    "\n",
    "def evaluate(model, iterator, criterion, tag_pad_idx):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            text = batch.text\n",
    "            tags = batch.tag\n",
    "            chars = batch.char\n",
    "            \n",
    "            predictions = model(text,chars)\n",
    "            \n",
    "            predictions = predictions.view(-1, predictions.shape[-1])\n",
    "            tags = tags.view(-1)\n",
    "            \n",
    "            loss = criterion(predictions, tags)\n",
    "            acc = accuracy_per_tag(predictions, tags)\n",
    "            f1 = f1_loss(predictions, tags, tag_pad_idx) \n",
    "            \n",
    "            epoch_loss += loss.item()\n",
    "            epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), acc, epoch_f1 / len(iterator)\n",
    "\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs\n",
    "\n",
    "N_EPOCHS = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss, train_acc, train_f1 = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n",
    "    valid_loss, valid_acc, valid_f1 = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'tut7-model.pt')\n",
    "    if epoch%2 == 0:\n",
    "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "        print(f'\\tTrain Loss: {train_loss:.3f} | Train F1 score: {train_f1*100:.2f}%')\n",
    "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1 score: {valid_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy of <pad>:  0% ( 0/776)\n",
      "Train Accuracy of     O: 100% (295/295)\n",
      "Train Accuracy of B-LOC:  0% ( 0/22)\n",
      "Train Accuracy of B-PER:  0% ( 0/16)\n",
      "Train Accuracy of B-ORG:  0% ( 0/ 8)\n",
      "Train Accuracy of I-PER:  0% ( 0/11)\n",
      "Train Accuracy of I-ORG:  0% ( 0/ 8)\n",
      "Train Accuracy of B-MISC:  0% ( 0/ 7)\n",
      "Train Accuracy of I-LOC:  0% ( 0/ 7)\n",
      "Train Accuracy of I-MISC:  0% ( 0/ 2)\n"
     ]
    }
   ],
   "source": [
    "n_tags = len(TAG.vocab)\n",
    "for i in range(n_tags):   \n",
    "    print('Train Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "           TAG.vocab.itos[i], train_acc[i][0],\n",
    "           train_acc[i][1], train_acc[i][2]))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid Accuracy of <pad>:  0% ( 0/230)\n",
      "Valid Accuracy of     O: 100% (148/148)\n",
      "Valid Accuracy of B-LOC:  0% ( 0/ 1)\n",
      "Valid Accuracy of B-PER:  0% ( 0/ 2)\n",
      "Valid Accuracy of B-ORG:  0% ( 0/ 7)\n",
      "Valid Accuracy of I-PER:  0% ( 0/ 2)\n",
      "Valid Accuracy of I-ORG:  0% ( 0/ 5)\n",
      "Valid Accuracy of B-MISC:  0% ( 0/ 1)\n",
      "Valid Accuracy of I-LOC: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-MISC: 100% ( 0/ 0)\n"
     ]
    }
   ],
   "source": [
    "for i in range(n_tags):\n",
    "    print('Valid Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "           TAG.vocab.itos[i], valid_acc[i][0],\n",
    "           valid_acc[i][1], valid_acc[i][2]))\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of <pad>:  0% ( 0/ 9)\n",
      "Test Accuracy of     O: 100% (84/84)\n",
      "Test Accuracy of B-LOC:  0% ( 0/ 4)\n",
      "Test Accuracy of B-PER:  0% ( 0/ 1)\n",
      "Test Accuracy of B-ORG: 100% ( 0/ 0)\n",
      "Test Accuracy of I-PER: 100% ( 0/ 0)\n",
      "Test Accuracy of I-ORG: 100% ( 0/ 0)\n",
      "Test Accuracy of B-MISC:  0% ( 0/ 3)\n",
      "Test Accuracy of I-LOC: 100% ( 0/ 0)\n",
      "Test Accuracy of I-MISC:  0% ( 0/ 1)\n",
      "Test Loss: 0.892 |  Test F1 score: 0.00%\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('tut7-model.pt'))\n",
    "\n",
    "test_loss, test_acc, test_f1 = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n",
    "n_tags = len(TAG.vocab)\n",
    "for i in range(n_tags):   \n",
    "    print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "           TAG.vocab.itos[i], test_acc[i][0],\n",
    "           test_acc[i][1], test_acc[i][2]))\n",
    "print(f'Test Loss: {test_loss:.3f} |  Test F1 score: {test_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tag_sentence(model, device, sentence, text_field, tag_field, char_field):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    if isinstance(sentence, str):\n",
    "        nlp = spacy.load('en')\n",
    "        tokens = [token.text for token in nlp(sentence)]\n",
    "    else:\n",
    "        tokens = [token for token in sentence]\n",
    "\n",
    "    if text_field.lower:\n",
    "        tokens = [t.lower() for t in tokens]\n",
    "        \n",
    "    max_word_len = max([len(token) for token in tokens])\n",
    "    numericalized_chars = []\n",
    "    char_pad_id = char_field.vocab.stoi[CHAR.pad_token] \n",
    "    for token in tokens:\n",
    "        numericalized_chars.append(\n",
    "                [char_field.vocab.stoi[char] for char in token]\n",
    "                + [char_pad_id for _ in range(max_word_len - len(token))]\n",
    "                )\n",
    "    numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n",
    "\n",
    "    unk_idx = text_field.vocab.stoi[text_field.unk_token]\n",
    "    \n",
    "    unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n",
    "    \n",
    "    token_tensor = torch.LongTensor(numericalized_tokens)\n",
    "    \n",
    "    token_tensor = token_tensor.unsqueeze(-1).to(device)\n",
    "    char_tensor = torch.as_tensor(numericalized_chars)\n",
    "    char_tensor = char_tensor.unsqueeze(0).to(device) \n",
    "    \n",
    "    predictions = model(token_tensor, char_tensor)\n",
    "    \n",
    "    top_predictions = predictions.argmax(-1)\n",
    "    \n",
    "    predicted_tags = [tag_field.vocab.itos[t.item()] for t in top_predictions]\n",
    "    \n",
    "    return tokens, predicted_tags, unks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"', 'we', 'do', \"n't\", 'support', 'any', 'such', 'recommendation', 'because', 'we', 'do', \"n't\", 'see', 'any', 'grounds', 'for', 'it', ',', '\"', 'the', 'commission', \"'s\", 'chief', 'spokesman', 'nikolaus', 'van', 'der', 'pas', 'told', 'a', 'news', 'briefing', '.']\n",
      "Pred. Tag\tActual Tag\tCorrect?\tToken\n",
      "\n",
      "O\t\tO\t\t✔\t\t\"\n",
      "O\t\tO\t\t✔\t\twe\n",
      "O\t\tO\t\t✔\t\tdo\n",
      "O\t\tO\t\t✔\t\tn't\n",
      "O\t\tO\t\t✔\t\tsupport\n",
      "O\t\tO\t\t✔\t\tany\n",
      "O\t\tO\t\t✔\t\tsuch\n",
      "O\t\tO\t\t✔\t\trecommendation\n",
      "O\t\tO\t\t✔\t\tbecause\n",
      "O\t\tO\t\t✔\t\twe\n",
      "O\t\tO\t\t✔\t\tdo\n",
      "O\t\tO\t\t✔\t\tn't\n",
      "O\t\tO\t\t✔\t\tsee\n",
      "O\t\tO\t\t✔\t\tany\n",
      "O\t\tO\t\t✔\t\tgrounds\n",
      "O\t\tO\t\t✔\t\tfor\n",
      "O\t\tO\t\t✔\t\tit\n",
      "O\t\tO\t\t✔\t\t,\n",
      "O\t\tO\t\t✔\t\t\"\n",
      "O\t\tO\t\t✔\t\tthe\n",
      "O\t\tB-ORG\t\t✘\t\tcommission\n",
      "O\t\tO\t\t✔\t\t's\n",
      "O\t\tO\t\t✔\t\tchief\n",
      "O\t\tO\t\t✔\t\tspokesman\n",
      "O\t\tB-PER\t\t✘\t\tnikolaus\n",
      "O\t\tI-PER\t\t✘\t\tvan\n",
      "O\t\tI-PER\t\t✘\t\tder\n",
      "O\t\tI-PER\t\t✘\t\tpas\n",
      "O\t\tO\t\t✔\t\ttold\n",
      "O\t\tO\t\t✔\t\ta\n",
      "O\t\tO\t\t✔\t\tnews\n",
      "O\t\tO\t\t✔\t\tbriefing\n",
      "O\t\tO\t\t✔\t\t.\n"
     ]
    }
   ],
   "source": [
    "example_index = 6\n",
    "\n",
    "sentence = vars(train_data.examples[example_index])['text']\n",
    "actual_tags = vars(train_data.examples[example_index])['tag']\n",
    "\n",
    "print(sentence)\n",
    "tokens, pred_tags, unks = tag_sentence(model, \n",
    "                                       device, \n",
    "                                       sentence, \n",
    "                                       TEXT, \n",
    "                                       TAG,\n",
    "                                       CHAR)\n",
    "\n",
    "print(\"Pred. Tag\\tActual Tag\\tCorrect?\\tToken\\n\")\n",
    "\n",
    "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n",
    "    correct = '✔' if pred_tag == actual_tag else '✘'\n",
    "    print(f\"{pred_tag}\\t\\t{actual_tag}\\t\\t{correct}\\t\\t{token}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'The will deliver a speech about the conflict in Sao Paulo at tomorrow in Anne Mary with Jack.'\n",
    "\n",
    "tokens, tags, unks = tag_sentence(model, \n",
    "                                  device, \n",
    "                                  sentence, \n",
    "                                  TEXT, \n",
    "                                  TAG,\n",
    "                                  CHAR)\n",
    "\n",
    "print(unks)\n",
    "print(\"Pred. Tag\\tToken\\n\")\n",
    "\n",
    "\n",
    "for token, tag in zip(tokens, tags):\n",
    "    print(f\"{tag}\\t\\t{token}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}