7-BERT-NER_(1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT\n",
    "\n",
    "Dans ce notebook on utilise les classes de PyTorch `TransformerEncoderLayer`, `TransformerEncoder` pour implémenter un modèle BERT pour la reconnaissance d'entités nommées.\n",
    "\n",
    "## Préparer les données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install torchtext==0.6.0\n",
    "\n",
    "import time\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.optim import Adam\n",
    "from torchtext.data import Field, BucketIterator\n",
    "from torchtext.datasets import SequenceTaggingDataset\n",
    "from torchtext.vocab import Vocab\n",
    "from collections import Counter\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "\n",
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import time\n",
    "import random\n",
    "import string\n",
    "from itertools import chain\n",
    "\n",
    "import numpy as np\n",
    "import copy\n",
    "from copy import deepcopy\n",
    "import torch.optim as optim\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "TEXT = data.Field(lower = True) \n",
    "TAG = data.Field(unk_token = None) \n",
    "\n",
    "my_path = \"../libex/data/\" \n",
    "my_path = \"./data_ner/\" \n",
    "\n",
    "train_data, valid_data, test_data = data.TabularDataset.splits(\n",
    "        path=my_path,\n",
    "        train=\"train.csv\",\n",
    "        validation=\"valid.csv\",\n",
    "        test=\"test.csv\", format='csv', skip_header=True,\n",
    "        fields=((\"text\", TEXT), (\"tag\", TAG))\n",
    "         \n",
    "    )\n",
    "\n",
    "MIN_FREQ = 2\n",
    "\n",
    "TEXT.build_vocab(train_data, \n",
    "                 min_freq = MIN_FREQ, # les mots qui apparaissent moins que MIN_FREQ fois seront ignorés du vocabulaire\n",
    "                 vectors = \"glove.6B.300d\",\n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "\n",
    "TAG.build_vocab(train_data)\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device = 'cpu'\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE,\n",
    "    device = device, sort=False)\n",
    "\n",
    "# padding index\n",
    "TEXT_PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token] \n",
    "TAG_PAD_IDX = TAG.vocab.stoi[TAG.pad_token]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construire le modèle\n",
    "\n",
    "Le modèle BERT est décrit dans [cet article](https://arxiv.org/abs/1810.04805) et est composé de N = 12 transformers composés de 6 encoders et d'une couche linéaire.\n",
    "\n",
    "Nous utilisons les paramètres du modèle BERT de l'article $BERT_{BASE}$ avec : \n",
    " \n",
    " - N = 12 transformers \n",
    " - Dimension des états cachés : fc_hidden = 768 \n",
    " - Têtes d'attention : attn_heads = 12\n",
    " \n",
    "La fonction `clones` suivante permet de produire N couches identiques et sera utile pour assembler N=12 modèles transformers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clones(module, N):\n",
    "    \"Produit N couches identiques\"\n",
    "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On implémente le même modèle transformer que dans un précédent notebook dans une classe à part."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "    def __init__(self, embedding_dim, attn_heads, dropout, trf_layers,fc_hidden, output_dim):\n",
    "        super(Transformer, self).__init__()\n",
    "          \n",
    "        encoder_layers = nn.TransformerEncoderLayer(\n",
    "            d_model=all_emb_size,\n",
    "            nhead=attn_heads,\n",
    "            activation=\"relu\",\n",
    "            dropout=dropout\n",
    "        )\n",
    "        self.encoder = nn.TransformerEncoder(\n",
    "            encoder_layer=encoder_layers,\n",
    "            num_layers=trf_layers\n",
    "        ) \n",
    "                \n",
    "        self.fc1 = nn.Linear(\n",
    "        in_features=all_emb_size,\n",
    "        out_features=fc_hidden\n",
    "        )\n",
    "        self.fc1_gelu = nn.GELU()\n",
    "        self.fc1_norm = nn.LayerNorm(fc_hidden)\n",
    "        self.fc2_dropout = nn.Dropout(dropout)\n",
    "        self.fc2 = nn.Linear(\n",
    "            in_features=fc_hidden,\n",
    "            out_features=output_dim\n",
    "        ) \n",
    "    def forward(self, words, pos_out, word_pad_idx):\n",
    "        \n",
    "        key_padding_mask = torch.as_tensor(words == word_pad_idx).permute(1, 0)\n",
    "        # pos_out = [sentence length, batch size, embedding dim ]\n",
    "        \n",
    "        # enc_out = [sentence length, batch size, embedding dim ]\n",
    "        enc_out = self.encoder(pos_out, src_key_padding_mask=key_padding_mask)\n",
    "        # Fully-connected\n",
    "        # fc1_out = [sentence length, batch size, fc hidden]\n",
    "        fc1_out = self.fc1_norm(self.fc1_gelu(self.fc1(enc_out)))\n",
    "        # fc2_out = [sentence length, batch size, output dim]\n",
    "        fc2_out = self.fc2(self.fc2_dropout(fc1_out))\n",
    "        return fc2_out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_masking(x):\n",
    "    # x: (batch_size, seq_len)\n",
    "    padded_positions = x == TEXT_PAD_IDX\n",
    "    return padded_positions.unsqueeze(1)\n",
    "\n",
    "class PositionalEmbedding(nn.Module):\n",
    "\n",
    "    def __init__(self, max_len, hidden_size, ):\n",
    "        super(PositionalEmbedding, self).__init__()\n",
    "        self.positional_embedding = nn.Embedding(max_len, hidden_size)\n",
    "        positions = torch.arange(0, max_len)\n",
    "        self.register_buffer('positions', positions)\n",
    "\n",
    "    def forward(self, sequence):\n",
    "        batch_size, seq_len = sequence.size()\n",
    "        positions = self.positions[:seq_len].unsqueeze(0).repeat(batch_size, 1)\n",
    "        return self.positional_embedding(positions)\n",
    "class PositionalEncoding(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, dropout=0.1, max_len=500):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)\n",
    "\n",
    "def build_model(layers_count, hidden_size, heads_count, d_ff, dropout, max_len, vocabulary_size):\n",
    "    \n",
    "    token_embedding = nn.Embedding(num_embeddings=vocabulary_size, embedding_dim=hidden_size)\n",
    "    positional_embedding = PositionalEncoding(max_len=max_len, hidden_size=hidden_size )\n",
    "\n",
    "    encoder_layers = nn.TransformerEncoderLayer(\n",
    "            d_model=hidden_size,\n",
    "            nhead=heads_count,\n",
    "            activation=\"relu\",\n",
    "            dropout=dropout)    \n",
    "    \n",
    "    \n",
    "    encoder = nn.TransformerEncoder(\n",
    "            encoder_layer=encoder_layers,\n",
    "            num_layers=layers_count\n",
    "        ) \n",
    "\n",
    "    bert = BERT(\n",
    "        encoder=encoder,\n",
    "        token_embedding=token_embedding,\n",
    "        positional_embedding=positional_embedding,\n",
    "        hidden_size=hidden_size,\n",
    "        vocabulary_size=vocabulary_size)\n",
    "\n",
    "    return bert\n",
    "\n",
    "class BERT(nn.Module):\n",
    "\n",
    "    def __init__(self, encoder, token_embedding, positional_embedding, hidden_size, vocabulary_size):\n",
    "        super(BERT, self).__init__()\n",
    "\n",
    "        self.encoder = encoder\n",
    "        self.token_embedding = token_embedding\n",
    "        self.positional_embedding = positional_embedding\n",
    "        self.token_prediction_layer = nn.Linear(hidden_size, vocabulary_size)\n",
    "        self.classification_layer = nn.Linear(hidden_size, len(TAG.vocab))\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        sequence = inputs\n",
    "        token_embedded = self.token_embedding(sequence)\n",
    "        positional_embedded = self.positional_embedding(token_embedded)\n",
    "        #embedded_sources = token_embedded + positional_embedded \n",
    "      \n",
    "        key_padding_mask = torch.as_tensor(sequence == TEXT_PAD_IDX).permute(1, 0)\n",
    "        encoded_sources = self.encoder(embedded_sources, key_padding_mask)\n",
    "        token_predictions = self.token_prediction_layer(encoded_sources)\n",
    "        classification_embedding = encoded_sources[:, 0, :]\n",
    "        classification_output = self.classification_layer(classification_embedding)\n",
    "        return token_predictions, classification_output\n",
    "    def count_parameters(self):\n",
    "        return sum(p.numel() for p in self.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "La classe BERT contient alors l'embedding des mots et des caractères et l'assemblage des N transformers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le modèle a 16,300,050 paramètres à entraîner.\n"
     ]
    }
   ],
   "source": [
    "model = build_model(hidden_size=300, layers_count=6, heads_count=4, d_ff=1024, dropout=0.1, max_len=512,\n",
    "                        vocabulary_size=len(TEXT.vocab))\n",
    "\n",
    "def init_weights(m):\n",
    "    for name, param in m.named_parameters():\n",
    "        nn.init.normal_(param.data, mean = 0, std = 0.1)\n",
    "        \n",
    "model.apply(init_weights)\n",
    "\n",
    "pretrained_embeddings = TEXT.vocab.vectors\n",
    "model.token_embedding.weight.data.copy_(pretrained_embeddings)\n",
    "model.token_embedding.weight.data[TAG_PAD_IDX] = torch.zeros( 300)\n",
    "\n",
    "print(f\"Le modèle a {model.count_parameters():,} paramètres à entraîner.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class BERT(nn.Module):\n",
    "\n",
    "    def __init__(self,\n",
    "                 input_dim,\n",
    "                 embedding_dim,\n",
    "                 attn_heads,\n",
    "                 fc_hidden,\n",
    "                 trf_layers,\n",
    "                 output_dim,\n",
    "                 dropout,\n",
    "                 word_pad_idx,\n",
    "                 tag_pad_idx,\n",
    "                 N):  \n",
    "        super().__init__()\n",
    "        self.N = N\n",
    "        self.word_pad_idx = word_pad_idx\n",
    "        self.tag_pad_idx = tag_pad_idx\n",
    "        # LAYER 1A: Word Embedding\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.embedding = nn.Embedding(\n",
    "            num_embeddings=input_dim,\n",
    "            embedding_dim=embedding_dim,\n",
    "            padding_idx=word_pad_idx\n",
    "        )\n",
    "        self.emb_dropout = nn.Dropout(dropout)\n",
    "        all_emb_size = embedding_dim \n",
    "        self.position_encoder = PositionalEncoding(\n",
    "            d_model=all_emb_size\n",
    "        )\n",
    "        \n",
    "        self.layers = clones(Transformer(embedding_dim, attn_heads, dropout, trf_layers,fc_hidden, output_dim), N)\n",
    "        #self.trans = Transformer(embedding_dim, char_emb_dim, char_cnn_filter_num, attn_heads, trf_dropout, trf_layers,fc_hidden, output_dim)\n",
    "\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "   \n",
    "        # init weights from normal distribution\n",
    "        for name, param in self.named_parameters():\n",
    "            nn.init.normal_(param.data, mean=0, std=0.1)\n",
    "\n",
    "    def forward(self, words, tags=None):\n",
    "        # words = [sentence length, batch size]\n",
    "        # tags = [sentence length, batch size]\n",
    "        # embedding_out = [sentence length, batch size, embedding dim]\n",
    "        embedding_out = self.emb_dropout(self.embedding(words))\n",
    "        pos_out = self.position_encoder(embedding_out)\n",
    "        for i in range(self.N):\n",
    "            x = self.layers[i](words, pos_out, self.word_pad_idx)    \n",
    "        return self.softmax(x)\n",
    "\n",
    "    def init_embeddings(self, word_pad_idx=TEXT_PAD_IDX,pretrained=None, freeze=True):\n",
    "        # initialize embedding for padding as zero\n",
    "        self.embedding.weight.data[self.word_pad_idx] = torch.zeros(self.embedding_dim)\n",
    "        if pretrained is not None:\n",
    "            self.embedding = nn.Embedding.from_pretrained(\n",
    "                embeddings=torch.as_tensor(pretrained),\n",
    "                padding_idx=self.word_pad_idx,\n",
    "                freeze=freeze\n",
    "            )\n",
    "    def count_parameters(self):\n",
    "        return sum(p.numel() for p in self.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le modèle a 22,099,584 paramètres à entraîner.\n"
     ]
    }
   ],
   "source": [
    "model = BERT(\n",
    "    input_dim=len(TEXT.vocab),\n",
    "    embedding_dim=300,\n",
    "    attn_heads=12,  \n",
    "    fc_hidden=768,  \n",
    "    trf_layers=1,\n",
    "    output_dim=len(TAG.vocab),\n",
    "    dropout=0.7,\n",
    "    word_pad_idx=TEXT_PAD_IDX,\n",
    "    tag_pad_idx=TAG_PAD_IDX,\n",
    "    N = 12\n",
    ")\n",
    "model.init_embeddings(\n",
    "    word_pad_idx=TEXT_PAD_IDX,\n",
    "    pretrained= TEXT.vocab.vectors,\n",
    "    freeze=True\n",
    ")\n",
    "\n",
    "print(f\"Le modèle a {model.count_parameters():,} paramètres à entraîner.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr = 5e-5)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy_per_tag(predictions, tags):\n",
    "    n_tags = len(TAG.vocab)\n",
    "    class_correct = list(0 for i in range(n_tags))\n",
    "    class_total = list(0 for i in range(n_tags))\n",
    "    acc = list(0 for i in range(n_tags))\n",
    "    _, pred = torch.max(predictions, 1)\n",
    "    # # compare predictions to true label\n",
    "    correct = np.squeeze(pred.eq(tags.data.view_as(pred)))\n",
    "    # # calculate test accuracy for each object class\n",
    "    for i in range(len(tags.data)):\n",
    "        label = tags.data[i]\n",
    "        class_correct[label] += correct[i].item()\n",
    "        class_total[label] += 1\n",
    "    for i in range(n_tags):\n",
    "        if np.sum(class_total[i]) == 0 and np.sum(class_correct[i]) ==0:\n",
    "            res = 100\n",
    "        else:\n",
    "            res = 100 * class_correct[i] / class_total[i]\n",
    "        acc[i] = res, np.sum(class_correct[i]), np.sum(class_total[i])\n",
    "        \n",
    "    return acc  \n",
    "\n",
    "from sklearn.metrics import f1_score\n",
    "def f1_loss(preds, y, tag_pad_idx):\n",
    "    index_o = TAG.vocab.stoi[\"O\"]\n",
    "    positive_labels = [i for i in range(len(TAG.vocab.itos))\n",
    "                           if i not in (tag_pad_idx, index_o)]\n",
    "    _, pred = torch.max(preds, 1)\n",
    "    pred = pred.data.cpu().numpy() \n",
    "    tags = y.data.cpu().numpy()\n",
    "    f1 = f1_score(\n",
    "            y_true=tags,\n",
    "            y_pred=pred,\n",
    "            labels=positive_labels,\n",
    "            average=\"micro\"\n",
    "        ) \n",
    "       \n",
    "    return f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "too many values to unpack (expected 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-90-b3b2963db989>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     79\u001b[0m     \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m     \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_f1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_iterator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTAG_PAD_IDX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m     \u001b[0mvalid_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_f1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_iterator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTAG_PAD_IDX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-90-b3b2963db989>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, iterator, optimizer, criterion, tag_pad_idx)\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;31m#predictions = [sent len, batch size, output dim]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-86-7969022e1701>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0msequence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0mtoken_embedded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoken_embedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0mpositional_embedded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpositional_embedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken_embedded\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m         \u001b[0;31m#embedded_sources = token_embedded + positional_embedded\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-86-7969022e1701>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sequence)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseq_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msequence\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mpositions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mseq_len\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepeat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpositional_embedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 2)"
     ]
    }
   ],
   "source": [
    "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        text = batch.text\n",
    "        tags = batch.tag\n",
    "      \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        predictions = model(text)\n",
    "        \n",
    "        #predictions = [sent len, batch size, output dim]\n",
    "        #tags = [sent len, batch size]\n",
    "        predictions = predictions.view(-1, predictions.shape[-1])\n",
    "        tags = tags.view(-1)\n",
    "        \n",
    "        #predictions = [sent len * batch size, output dim]\n",
    "        #tags = [sent len * batch size]\n",
    "        \n",
    "        loss = criterion(predictions, tags)\n",
    "                \n",
    "        f1 = f1_loss(predictions, tags, tag_pad_idx)\n",
    "        acc = accuracy_per_tag(predictions, tags)   \n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), acc, epoch_f1 / len(iterator)\n",
    "\n",
    "def evaluate(model, iterator, criterion, tag_pad_idx):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            text = batch.text\n",
    "            tags = batch.tag\n",
    "         \n",
    "            \n",
    "            predictions = model(text)\n",
    "            \n",
    "            predictions = predictions.view(-1, predictions.shape[-1])\n",
    "            tags = tags.view(-1)\n",
    "            \n",
    "            loss = criterion(predictions, tags)\n",
    "            acc = accuracy_per_tag(predictions, tags)\n",
    "            f1 = f1_loss(predictions, tags, tag_pad_idx) \n",
    "            \n",
    "            epoch_loss += loss.item()\n",
    "            epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), acc, epoch_f1 / len(iterator)\n",
    "\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs\n",
    "\n",
    "N_EPOCHS = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss, train_acc, train_f1 = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n",
    "    valid_loss, valid_acc, valid_f1 = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'tut7-model.pt')\n",
    "    if epoch%2 == 0:\n",
    "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "        print(f'\\tTrain Loss: {train_loss:.3f} | Train F1 score: {train_f1*100:.2f}%')\n",
    "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1 score: {valid_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_acc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-91-555949d68954>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_tags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     print('Train Accuracy of %5s: %2d%% (%2d/%2d)' % (\n\u001b[0;32m----> 4\u001b[0;31m            \u001b[0mTAG\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_acc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m            train_acc[i][1], train_acc[i][2]))  \n",
      "\u001b[0;31mNameError\u001b[0m: name 'train_acc' is not defined"
     ]
    }
   ],
   "source": [
    "n_tags = len(TAG.vocab)\n",
    "for i in range(n_tags):   \n",
    "    print('Train Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "           TAG.vocab.itos[i], train_acc[i][0],\n",
    "           train_acc[i][1], train_acc[i][2]))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid Accuracy of <pad>:  0% ( 0/77)\n",
      "Valid Accuracy of     O: 80% (28/35)\n",
      "Valid Accuracy of I-Locution: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-Catalyst: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-Support: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-Locution: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-Pore_volume: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-Pore_volume: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-PV_unit: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-Surface_area: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-SA_unit: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-Support: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-Surface_area: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-SA_val_min: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-PV_val_min: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-Catalyst: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-SA_val_max: 100% ( 0/ 0)\n",
      "Valid Accuracy of I-PV_val_max: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-PV_unit: 100% ( 0/ 0)\n",
      "Valid Accuracy of B-SA_unit: 100% ( 0/ 0)\n"
     ]
    }
   ],
   "source": [
    "for i in range(n_tags):\n",
    "    print('Valid Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "           TAG.vocab.itos[i], valid_acc[i][0],\n",
    "           valid_acc[i][1], valid_acc[i][2]))\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of <pad>:  0% ( 0/187)\n",
      "Test Accuracy of     O: 98% (106/108)\n",
      "Test Accuracy of I-Locution:  0% ( 0/ 2)\n",
      "Test Accuracy of I-Catalyst: 100% ( 0/ 0)\n",
      "Test Accuracy of I-Support:  0% ( 0/ 1)\n",
      "Test Accuracy of B-Locution: 100% ( 0/ 0)\n",
      "Test Accuracy of I-Pore_volume:  0% ( 0/ 1)\n",
      "Test Accuracy of B-Pore_volume:  0% ( 0/ 1)\n",
      "Test Accuracy of I-PV_unit:  0% ( 0/ 2)\n",
      "Test Accuracy of I-Surface_area: 100% ( 0/ 0)\n",
      "Test Accuracy of I-SA_unit: 100% ( 0/ 0)\n",
      "Test Accuracy of B-Support:  0% ( 0/ 1)\n",
      "Test Accuracy of B-Surface_area: 100% ( 0/ 0)\n",
      "Test Accuracy of I-SA_val_min: 100% ( 0/ 0)\n",
      "Test Accuracy of I-PV_val_min:  0% ( 0/ 1)\n",
      "Test Accuracy of B-Catalyst: 100% ( 0/ 0)\n",
      "Test Accuracy of I-SA_val_max: 100% ( 0/ 0)\n",
      "Test Accuracy of I-PV_val_max:  0% ( 0/ 1)\n",
      "Test Accuracy of B-PV_unit: 100% ( 0/ 0)\n",
      "Test Accuracy of B-SA_unit: 100% ( 0/ 0)\n",
      "Test Loss: 2.845 |  Test F1 score: 0.00%\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('tut7-model.pt'))\n",
    "\n",
    "test_loss, test_acc, test_f1 = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n",
    "n_tags = len(TAG.vocab)\n",
    "for i in range(n_tags):   \n",
    "    print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "           TAG.vocab.itos[i], test_acc[i][0],\n",
    "           test_acc[i][1], test_acc[i][2]))\n",
    "print(f'Test Loss: {test_loss:.3f} |  Test F1 score: {test_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tag_sentence(model, device, sentence, text_field, tag_field):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    if isinstance(sentence, str):\n",
    "        nlp = spacy.load('en')\n",
    "        tokens = [token.text for token in nlp(sentence)]\n",
    "    else:\n",
    "        tokens = [token for token in sentence]\n",
    "\n",
    "    if text_field.lower:\n",
    "        tokens = [t.lower() for t in tokens]\n",
    "        \n",
    "    max_word_len = max([len(token) for token in tokens])\n",
    "\n",
    "    numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n",
    "\n",
    "    unk_idx = text_field.vocab.stoi[text_field.unk_token]\n",
    "    \n",
    "    unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n",
    "    \n",
    "    token_tensor = torch.LongTensor(numericalized_tokens)\n",
    "    \n",
    "    token_tensor = token_tensor.unsqueeze(-1).to(device)\n",
    "    \n",
    "    predictions = model(token_tensor)\n",
    "    \n",
    "    top_predictions = predictions.argmax(-1)\n",
    "    \n",
    "    predicted_tags = [tag_field.vocab.itos[t.item()] for t in top_predictions]\n",
    "    \n",
    "    return tokens, predicted_tags, unks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['53', '.', 'the', 'method', 'of', 'claim', '3', 'wherein', 'the', 'silica', 'support', 'has', 'a', 'pore', 'volume', 'between', 'about', '1.0', 'cm3/g', 'and', 'about', '1.5', 'cm3/g', '.']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-Support', 'I-Support', 'O', 'O', 'B-Pore_volume', 'I-Pore_volume', 'I-Locution', 'O', 'I-PV_val_min', 'I-PV_unit', 'I-Locution', 'O', 'I-PV_val_max', 'I-PV_unit', 'O']\n",
      "['B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit', 'B-SA_unit']\n"
     ]
    }
   ],
   "source": [
    "example_index = 39\n",
    "\n",
    "sentence = vars(test_data.examples[example_index])['text']\n",
    "actual_tags = vars(test_data.examples[example_index])['tag']\n",
    "\n",
    "print(sentence)\n",
    "print(actual_tags)\n",
    "\n",
    "tokens, pred_tags, unks = tag_sentence(model, \n",
    "                                       device, \n",
    "                                       sentence, \n",
    "                                       TEXT, \n",
    "                                       TAG\n",
    "                                      )\n",
    "print(pred_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pred. Tag\tActual Tag\tCorrect?\tToken\n",
      "\n",
      "B-SA_unit\t\tO\t\t✘\t\t53\n",
      "B-SA_unit\t\tO\t\t✘\t\t.\n",
      "B-SA_unit\t\tO\t\t✘\t\tthe\n",
      "B-SA_unit\t\tO\t\t✘\t\tmethod\n",
      "B-SA_unit\t\tO\t\t✘\t\tof\n",
      "B-SA_unit\t\tO\t\t✘\t\tclaim\n",
      "B-SA_unit\t\tO\t\t✘\t\t3\n",
      "B-SA_unit\t\tO\t\t✘\t\twherein\n",
      "B-SA_unit\t\tO\t\t✘\t\tthe\n",
      "B-SA_unit\t\tB-Support\t\t✘\t\tsilica\n",
      "B-SA_unit\t\tI-Support\t\t✘\t\tsupport\n",
      "B-SA_unit\t\tO\t\t✘\t\thas\n",
      "B-SA_unit\t\tO\t\t✘\t\ta\n",
      "B-SA_unit\t\tB-Pore_volume\t\t✘\t\tpore\n",
      "B-SA_unit\t\tI-Pore_volume\t\t✘\t\tvolume\n",
      "B-SA_unit\t\tI-Locution\t\t✘\t\tbetween\n",
      "B-SA_unit\t\tO\t\t✘\t\tabout\n",
      "B-SA_unit\t\tI-PV_val_min\t\t✘\t\t1.0\n",
      "B-SA_unit\t\tI-PV_unit\t\t✘\t\tcm3/g\n",
      "B-SA_unit\t\tI-Locution\t\t✘\t\tand\n",
      "B-SA_unit\t\tO\t\t✘\t\tabout\n",
      "B-SA_unit\t\tI-PV_val_max\t\t✘\t\t1.5\n",
      "B-SA_unit\t\tI-PV_unit\t\t✘\t\tcm3/g\n",
      "B-SA_unit\t\tO\t\t✘\t\t.\n"
     ]
    }
   ],
   "source": [
    "print(\"Pred. Tag\\tActual Tag\\tCorrect?\\tToken\\n\")\n",
    "\n",
    "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n",
    "    correct = '✔' if pred_tag == actual_tag else '✘'\n",
    "    print(f\"{pred_tag}\\t\\t{actual_tag}\\t\\t{correct}\\t\\t{token}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'The will deliver a speech about the conflict in Sao Paulo at tomorrow in Anne Mary with Jack.'\n",
    "\n",
    "tokens, tags, unks = tag_sentence(model, \n",
    "                                  device, \n",
    "                                  sentence, \n",
    "                                  TEXT, \n",
    "                                  TAG,\n",
    "                                  CHAR)\n",
    "\n",
    "print(unks)\n",
    "print(\"Pred. Tag\\tToken\\n\")\n",
    "\n",
    "\n",
    "for token, tag in zip(tokens, tags):\n",
    "    print(f\"{tag}\\t\\t{token}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}