POS-Prefix-Classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "import bclm\n",
    "\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import classification_report\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertForTokenClassification, BertAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(3)\n",
    "np.random.seed(3)\n",
    "torch.cuda.manual_seed_all(3)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = bclm.read_dataframe('spmrl', subset='train')\n",
    "train_df = bclm.get_token_df(train, ['upostag'])\n",
    "train_df['token_str'] = train_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "dev = bclm.read_dataframe('spmrl', subset='dev')\n",
    "dev_df = bclm.get_token_df(dev, ['upostag'])\n",
    "dev_df['token_str'] = dev_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "test = bclm.read_dataframe('spmrl', subset='test')\n",
    "test_df = bclm.get_token_df(test, ['upostag'])\n",
    "test_df['token_str'] = test_df['token_str'].str.replace('”','\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_df['prefix'] = ''\n",
    "train_df['host'] = ''\n",
    "\n",
    "dev_df['prefix'] = ''\n",
    "dev_df['host'] = ''\n",
    "\n",
    "test_df['prefix'] = ''\n",
    "test_df['host'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>upostag</th>\n",
       "      <th>set</th>\n",
       "      <th>prefix</th>\n",
       "      <th>host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>CDT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>BN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>PREPOSITION^NNP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>לישראל</td>\n",
       "      <td>PREPOSITION^NNP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>כשהם</td>\n",
       "      <td>TEMP^PRP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>נרשמים</td>\n",
       "      <td>BN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>כמתנדבים</td>\n",
       "      <td>PREPOSITION^NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>אך</td>\n",
       "      <td>CC</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>למעשה</td>\n",
       "      <td>RB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>משמשים</td>\n",
       "      <td>BN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>עובדים</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>שכירים</td>\n",
       "      <td>JJ</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>זולים</td>\n",
       "      <td>JJ</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>.</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>תופעה</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>זו</td>\n",
       "      <td>PRP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>התבררה</td>\n",
       "      <td>VB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>אתמול</td>\n",
       "      <td>RB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>בוועדת</td>\n",
       "      <td>PREPOSITION^NNT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>העבודה</td>\n",
       "      <td>DEF^NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>והרווחה</td>\n",
       "      <td>CONJ^DEF^NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>של</td>\n",
       "      <td>POS</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>הכנסת</td>\n",
       "      <td>DEF^NNP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>שדנה</td>\n",
       "      <td>REL^VB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>בנושא</td>\n",
       "      <td>PREPOSITION^NNT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>העסקת</td>\n",
       "      <td>NNT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>עובדים</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sent_id  token_id token_str          upostag  set prefix host\n",
       "0         1         1     עשרות              CDT  dev            \n",
       "1         1         2     אנשים               NN  dev            \n",
       "2         1         3    מגיעים               BN  dev            \n",
       "3         1         4   מתאילנד  PREPOSITION^NNP  dev            \n",
       "4         1         5    לישראל  PREPOSITION^NNP  dev            \n",
       "5         1         6      כשהם         TEMP^PRP  dev            \n",
       "6         1         7    נרשמים               BN  dev            \n",
       "7         1         8  כמתנדבים   PREPOSITION^NN  dev            \n",
       "8         1         9         ,             yyCM  dev            \n",
       "9         1        10        אך               CC  dev            \n",
       "10        1        11     למעשה               RB  dev            \n",
       "11        1        12    משמשים               BN  dev            \n",
       "12        1        13    עובדים               NN  dev            \n",
       "13        1        14    שכירים               JJ  dev            \n",
       "14        1        15     זולים               JJ  dev            \n",
       "15        1        16         .            yyDOT  dev            \n",
       "16        2         1     תופעה               NN  dev            \n",
       "17        2         2        זו              PRP  dev            \n",
       "18        2         3    התבררה               VB  dev            \n",
       "19        2         4     אתמול               RB  dev            \n",
       "20        2         5    בוועדת  PREPOSITION^NNT  dev            \n",
       "21        2         6    העבודה           DEF^NN  dev            \n",
       "22        2         7   והרווחה      CONJ^DEF^NN  dev            \n",
       "23        2         8        של              POS  dev            \n",
       "24        2         9     הכנסת          DEF^NNP  dev            \n",
       "25        2        10         ,             yyCM  dev            \n",
       "26        2        11      שדנה           REL^VB  dev            \n",
       "27        2        12     בנושא  PREPOSITION^NNT  dev            \n",
       "28        2        13     העסקת              NNT  dev            \n",
       "29        2        14    עובדים               NN  dev            "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prefix_corretion(prefix):\n",
    "    not_prefixes = ['VB^POS','VB^DUMMY_AT','VB^AT', 'IN^DUMMY_AT', 'NN', 'IN', 'P',\n",
    "                   'POS', 'RB', 'AT']\n",
    "    first_prefix = ['DEF^BN^AT', 'CONJ^VB^AT','CONJ^BN^AT', 'CONJ^BN^AT', 'REL^VB^AT',\n",
    "                   'PREPOSITION^RB', 'DEF^yyQUOT', 'IN^yyQUOT', 'CONJ^yyQUOT^DEF',\n",
    "                   'PREPOSITION^yyQUOT^PREPOSITION^DEF', 'REL^yyQUOT', 'REL^IN', 'CONJ^IN',\n",
    "                   'PREPOSITION^yyQUOT', 'PREPOSITION^POS', 'PREPOSITION^IN', 'PREPOSITION^yyQUOT^DEF',\n",
    "                   'REL^yyQUOT^PREPOSITION', 'CONJ^yyQUOT', 'REL^AT']\n",
    "    \n",
    "    two_first_prefixes = ['CONJ^PREPOSITION^yyQUOT', 'PREPOSITION^DEF^yyQUOT', 'CONJ^DEF^yyQUOT']\n",
    "    \n",
    "    if prefix in not_prefixes:\n",
    "        host = prefix\n",
    "        prefix = '-'\n",
    "    \n",
    "    elif prefix in first_prefix:\n",
    "        tag_list = prefix.split('^')\n",
    "        prefix = tag_list[0]\n",
    "        host = '^'.join(tag_list[1:])\n",
    "        \n",
    "    elif prefix in two_first_prefixes:\n",
    "        tag_list = prefix.split('^')\n",
    "        prefix = '^'.join(tag_list[0:2])\n",
    "        host = '^'.join(tag_list[2:])\n",
    "        \n",
    "    else:\n",
    "        host = \"\"\n",
    "        prefix = prefix\n",
    "        \n",
    "    return prefix, host\n",
    "        \n",
    "\n",
    "def full_tag_to_prefix_host(df):\n",
    "    for index in df.index:\n",
    "        full_tag = df.at[index, 'upostag']\n",
    "        full_tag_list = full_tag.split('^')\n",
    "        if len(full_tag_list) > 1:\n",
    "            prefix = '^'.join(full_tag_list[:-1])\n",
    "            prefix, host = prefix_corretion(prefix)\n",
    "            df.at[index, 'prefix'] = prefix\n",
    "            if len(host) > 0:\n",
    "                host += '^'\n",
    "            host += full_tag_list[-1]\n",
    "            df.at[index, 'host'] = host\n",
    "        else:\n",
    "            df.at[index, 'prefix'] = '-'\n",
    "            df.at[index, 'host'] = full_tag_list[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>upostag</th>\n",
       "      <th>set</th>\n",
       "      <th>prefix</th>\n",
       "      <th>host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>93474</th>\n",
       "      <td>5436</td>\n",
       "      <td>15</td>\n",
       "      <td>נעליים</td>\n",
       "      <td>NN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93475</th>\n",
       "      <td>5436</td>\n",
       "      <td>16</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93476</th>\n",
       "      <td>5436</td>\n",
       "      <td>17</td>\n",
       "      <td>איך</td>\n",
       "      <td>QW</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>QW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93477</th>\n",
       "      <td>5436</td>\n",
       "      <td>18</td>\n",
       "      <td>מנהלים</td>\n",
       "      <td>BN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>BN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93478</th>\n",
       "      <td>5436</td>\n",
       "      <td>19</td>\n",
       "      <td>חשבון</td>\n",
       "      <td>NN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93479</th>\n",
       "      <td>5436</td>\n",
       "      <td>20</td>\n",
       "      <td>בבנק</td>\n",
       "      <td>PREPOSITION^DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>PREPOSITION^DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93480</th>\n",
       "      <td>5436</td>\n",
       "      <td>21</td>\n",
       "      <td>.</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyDOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93481</th>\n",
       "      <td>5437</td>\n",
       "      <td>1</td>\n",
       "      <td>אילנה</td>\n",
       "      <td>NNP</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93482</th>\n",
       "      <td>5437</td>\n",
       "      <td>2</td>\n",
       "      <td>נחום</td>\n",
       "      <td>NNP</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93483</th>\n",
       "      <td>5437</td>\n",
       "      <td>3</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93484</th>\n",
       "      <td>5437</td>\n",
       "      <td>4</td>\n",
       "      <td>מזכירת</td>\n",
       "      <td>NNT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93485</th>\n",
       "      <td>5437</td>\n",
       "      <td>5</td>\n",
       "      <td>הקיבוץ</td>\n",
       "      <td>DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93486</th>\n",
       "      <td>5437</td>\n",
       "      <td>6</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93487</th>\n",
       "      <td>5437</td>\n",
       "      <td>7</td>\n",
       "      <td>נתנה</td>\n",
       "      <td>VB</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>VB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93488</th>\n",
       "      <td>5437</td>\n",
       "      <td>8</td>\n",
       "      <td>ביטוי</td>\n",
       "      <td>NN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93489</th>\n",
       "      <td>5437</td>\n",
       "      <td>9</td>\n",
       "      <td>לחרדות</td>\n",
       "      <td>PREPOSITION^DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>PREPOSITION^DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93490</th>\n",
       "      <td>5437</td>\n",
       "      <td>10</td>\n",
       "      <td>של</td>\n",
       "      <td>POS</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>POS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93491</th>\n",
       "      <td>5437</td>\n",
       "      <td>11</td>\n",
       "      <td>בני</td>\n",
       "      <td>NNT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93492</th>\n",
       "      <td>5437</td>\n",
       "      <td>12</td>\n",
       "      <td>הקיבוץ</td>\n",
       "      <td>DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93493</th>\n",
       "      <td>5437</td>\n",
       "      <td>13</td>\n",
       "      <td>:</td>\n",
       "      <td>yyCLN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCLN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93494</th>\n",
       "      <td>5437</td>\n",
       "      <td>14</td>\n",
       "      <td>\"</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyQUOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93495</th>\n",
       "      <td>5437</td>\n",
       "      <td>15</td>\n",
       "      <td>זה</td>\n",
       "      <td>PRP</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>PRP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93496</th>\n",
       "      <td>5437</td>\n",
       "      <td>16</td>\n",
       "      <td>רע</td>\n",
       "      <td>JJ</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>JJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93497</th>\n",
       "      <td>5437</td>\n",
       "      <td>17</td>\n",
       "      <td>מאוד</td>\n",
       "      <td>RB</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93498</th>\n",
       "      <td>5437</td>\n",
       "      <td>18</td>\n",
       "      <td>אם</td>\n",
       "      <td>CC</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>CC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93499</th>\n",
       "      <td>5437</td>\n",
       "      <td>19</td>\n",
       "      <td>יגיעו</td>\n",
       "      <td>VB</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>VB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93500</th>\n",
       "      <td>5437</td>\n",
       "      <td>20</td>\n",
       "      <td>אלינו</td>\n",
       "      <td>IN^S_PRN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>IN^S_PRN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93501</th>\n",
       "      <td>5437</td>\n",
       "      <td>21</td>\n",
       "      <td>מכורח</td>\n",
       "      <td>PREPOSITION^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93502</th>\n",
       "      <td>5437</td>\n",
       "      <td>22</td>\n",
       "      <td>\"</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyQUOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93503</th>\n",
       "      <td>5437</td>\n",
       "      <td>23</td>\n",
       "      <td>.</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyDOT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sent_id  token_id token_str             upostag    set  \\\n",
       "93474     5436        15    נעליים                  NN  train   \n",
       "93475     5436        16         ,                yyCM  train   \n",
       "93476     5436        17       איך                  QW  train   \n",
       "93477     5436        18    מנהלים                  BN  train   \n",
       "93478     5436        19     חשבון                  NN  train   \n",
       "93479     5436        20      בבנק  PREPOSITION^DEF^NN  train   \n",
       "93480     5436        21         .               yyDOT  train   \n",
       "93481     5437         1     אילנה                 NNP  train   \n",
       "93482     5437         2      נחום                 NNP  train   \n",
       "93483     5437         3         ,                yyCM  train   \n",
       "93484     5437         4    מזכירת                 NNT  train   \n",
       "93485     5437         5    הקיבוץ              DEF^NN  train   \n",
       "93486     5437         6         ,                yyCM  train   \n",
       "93487     5437         7      נתנה                  VB  train   \n",
       "93488     5437         8     ביטוי                  NN  train   \n",
       "93489     5437         9    לחרדות  PREPOSITION^DEF^NN  train   \n",
       "93490     5437        10        של                 POS  train   \n",
       "93491     5437        11       בני                 NNT  train   \n",
       "93492     5437        12    הקיבוץ              DEF^NN  train   \n",
       "93493     5437        13         :               yyCLN  train   \n",
       "93494     5437        14         \"              yyQUOT  train   \n",
       "93495     5437        15        זה                 PRP  train   \n",
       "93496     5437        16        רע                  JJ  train   \n",
       "93497     5437        17      מאוד                  RB  train   \n",
       "93498     5437        18        אם                  CC  train   \n",
       "93499     5437        19     יגיעו                  VB  train   \n",
       "93500     5437        20     אלינו            IN^S_PRN  train   \n",
       "93501     5437        21     מכורח      PREPOSITION^NN  train   \n",
       "93502     5437        22         \"              yyQUOT  train   \n",
       "93503     5437        23         .               yyDOT  train   \n",
       "\n",
       "                prefix      host  \n",
       "93474                -        NN  \n",
       "93475                -      yyCM  \n",
       "93476                -        QW  \n",
       "93477                -        BN  \n",
       "93478                -        NN  \n",
       "93479  PREPOSITION^DEF        NN  \n",
       "93480                -     yyDOT  \n",
       "93481                -       NNP  \n",
       "93482                -       NNP  \n",
       "93483                -      yyCM  \n",
       "93484                -       NNT  \n",
       "93485              DEF        NN  \n",
       "93486                -      yyCM  \n",
       "93487                -        VB  \n",
       "93488                -        NN  \n",
       "93489  PREPOSITION^DEF        NN  \n",
       "93490                -       POS  \n",
       "93491                -       NNT  \n",
       "93492              DEF        NN  \n",
       "93493                -     yyCLN  \n",
       "93494                -    yyQUOT  \n",
       "93495                -       PRP  \n",
       "93496                -        JJ  \n",
       "93497                -        RB  \n",
       "93498                -        CC  \n",
       "93499                -        VB  \n",
       "93500                -  IN^S_PRN  \n",
       "93501      PREPOSITION        NN  \n",
       "93502                -    yyQUOT  \n",
       "93503                -     yyDOT  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_tag_to_prefix_host(train_df)\n",
    "full_tag_to_prefix_host(dev_df)\n",
    "full_tag_to_prefix_host(test_df)\n",
    "\n",
    "train_df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'PAD': 0, 'CONJ^PREPOSITION': 1, 'TEMP': 2, 'ZVL^PREPOSITION': 3, 'IN': 4, 'IN^RB': 5, 'PREPOSITION^REL': 6, 'TEMP^PREPOSITION': 7, 'CONJ^REL': 8, 'TEMP^PREPOSITION^DEF': 9, 'ZVL^DEF': 10, 'DEF^DEF': 11, 'ZVL': 12, 'DEF': 13, 'CONJ^IN^DEF': 14, 'REL^DEF': 15, 'PREPOSITIONIN': 16, 'REL^PREPOSITION^yyQUOT': 17, 'PREPOSITION^PREPOSITION': 18, 'ZVL^PREPOSITION^DEF': 19, 'REL^yyQUOT^DEF': 20, 'PREPOSITION^IN^DEF': 21, 'ADVERB': 22, 'IN^DEF': 23, 'REL': 24, 'CC^ZVL^DEF': 25, 'REL^ADVERB': 26, 'IN^IN': 27, 'PREPOSITION^DEF': 28, 'CONJ': 29, 'PREPOSITION^ADVERB': 30, 'CONJ^REL^PREPOSITION': 31, '-': 32, 'PREPOSITION^ PREPOSITION^DEF': 33, 'REL^PREPOSITION': 34, 'IN^REL': 35, 'CONJ^TEMP': 36, 'CONJ^REL^DEF': 37, 'TEMP^DEF': 38, 'DEF^PREPOSITION': 39, 'PREPOSITION': 40, 'CONJ^PREPOSITION^DEF': 41, 'IN^IN^DEF': 42, 'PREPOSITION^yyQUOT^PREPOSITION': 43, 'PREPOSITIONIN^PREPOSITION': 44, 'PREPOSITION^PREPOSITION^DEF': 45, 'CONJ^DEF': 46, 'REL^PREPOSITION^DEF': 47}\n",
      "48\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([train_df, dev_df, test_df])\n",
    "data.head()\n",
    "tag_vals = list(set(data[\"prefix\"].values))\n",
    "tags = ['PAD'] + tag_vals\n",
    "tag2idx = {tag:idx for idx, tag in enumerate(tags)}\n",
    "idx2tag = {idx:tag for idx, tag in enumerate(tags)}\n",
    "\n",
    "print(tag2idx)\n",
    "# print(idx2tag)\n",
    "print(len(tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for label in tag_vals:\n",
    "#     print(data[data['prefix'] == 'PREPOSITIONIN'])\n",
    "    \n",
    "data[data['prefix'] == 'PREPOSITIONIN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['הם', 'התבקשו', 'לדווח', 'למשטרה', 'על', 'תנועותיהם', '.']\n",
      "['-', '-', '-', 'PREPOSITION^DEF', '-', '-', '-']\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "class sentenceGetter(object):\n",
    "    def __init__(self, data, max_sent=None):\n",
    "        self.index = 0\n",
    "        self.max_sent = max_sent\n",
    "        self.tokens = data['token_str']\n",
    "        self.labels = data['prefix']\n",
    "        #for evaluating by word-accuracy\n",
    "        self.correspondingToken = data['token_id']\n",
    "        self.orig_sent_id = data['sent_id']\n",
    "    \n",
    "    def sentences(self):\n",
    "        sent = []\n",
    "        counter = 0\n",
    "        \n",
    "        for token,label, corres_tok, sent_id in zip(self.tokens, self.labels, self.correspondingToken, self.orig_sent_id):\n",
    "            sent.append((token, label, corres_tok, sent_id))\n",
    "            if token.strip() == \".\":\n",
    "                yield sent\n",
    "                sent = []\n",
    "                counter += 1\n",
    "            if self.max_sent is not None and counter >= self.max_sent:\n",
    "                return\n",
    "\n",
    "train_getter = sentenceGetter(train_df)\n",
    "dev_getter = sentenceGetter(dev_df)\n",
    "test_getter = sentenceGetter(test_df)\n",
    "\n",
    "train_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "train_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "\n",
    "dev_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "\n",
    "test_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "\n",
    "print(train_sentences[10])\n",
    "print(train_labels[10])\n",
    "\n",
    "print(len(dev_sentences))\n",
    "\n",
    "del dev_sentences[296]\n",
    "del dev_labels[296]\n",
    "del dev_corresTokens[296]\n",
    "del dev_sent_ids[296]\n",
    "\n",
    "\n",
    "del dev_sentences[226]\n",
    "del dev_labels[226]\n",
    "del dev_corresTokens[226]\n",
    "del dev_sent_ids[226]\n",
    "\n",
    "\n",
    "del dev_sentences[57]\n",
    "del dev_labels[57]\n",
    "del dev_corresTokens[57]\n",
    "del dev_sent_ids[57]\n",
    "\n",
    "\n",
    "del dev_sentences[49]\n",
    "del dev_labels[49]\n",
    "del dev_corresTokens[49]\n",
    "del dev_sent_ids[49]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda\n",
      "Number of gpus: 4\n",
      "Name of gpu: GeForce GTX 1080 Ti\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.set_device(1)\n",
    "\n",
    "print(\"Device: \" + str(device))\n",
    "print(\"Number of gpus: \" + str(n_gpu))\n",
    "print(\"Name of gpu: \" + torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 150\n",
    "bs = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The pre-trained model you are loading is a cased model but you have not set `do_lower_case` to False. We are setting `do_lower_case=False` for you but you may want to check this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['הם', 'ה', '##ת', '##בק', '##שו', 'ל', '##דו', '##וח', 'ל', '##משטרה', 'על', 'ת', '##נוע', '##ות', '##יהם', '.']\n",
      "['-', '-', '-', '-', '-', '-', '-', '-', 'PREPOSITION^DEF', 'PREPOSITION^DEF', '-', '-', '-', '-', '-', '-']\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
    "def tokenize(sentences, orig_labels):\n",
    "    tokenized_texts = []\n",
    "    labels = []\n",
    "    for sent, sent_labels in zip(sentences, orig_labels):\n",
    "        bert_tokens = []\n",
    "        bert_labels = []\n",
    "        for orig_token, orig_label in zip(sent, sent_labels):\n",
    "            b_tokens = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(b_tokens)\n",
    "            for b_token in b_tokens:\n",
    "                bert_labels.append(orig_label)\n",
    "        tokenized_texts.append(bert_tokens)\n",
    "        labels.append(bert_labels)\n",
    "        assert len(bert_tokens) == len(bert_labels)\n",
    "    return tokenized_texts, labels\n",
    "\n",
    "train_tokenized_texts, train_tokenized_labels = tokenize(train_sentences, train_labels)\n",
    "print(train_tokenized_texts[10])\n",
    "print(train_tokenized_labels[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentences_and_labels(tokenized_texts, labels):\n",
    "    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                              maxlen = MAX_LEN, dtype = \"float32\", truncating = \"post\", padding = \"post\", value = tag2idx['PAD'])\n",
    "    tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels], \n",
    "                         maxlen = MAX_LEN, value = tag2idx['PAD'], padding = \"post\",\n",
    "                        dtype = \"float32\", truncating = \"post\")\n",
    "    attention_masks = [[float(i>0) for i in ii] for ii in input_ids]\n",
    "    return input_ids, tags, attention_masks\n",
    "\n",
    "input_ids, tags, attention_masks = pad_sentences_and_labels(train_tokenized_texts, train_tokenized_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "tr_tags = torch.tensor(tags, dtype=torch.long)\n",
    "tr_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "# train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, batch_size=bs, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   7%|▋         | 1/15 [01:24<19:48, 84.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.6085739363180963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  13%|█▎        | 2/15 [02:54<18:42, 86.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.1686880534122649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  20%|██        | 3/15 [04:24<17:27, 87.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.10560035813403756\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  27%|██▋       | 4/15 [05:52<16:04, 87.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.07537848825909589\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  33%|███▎      | 5/15 [07:20<14:37, 87.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.054457569531606215\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  40%|████      | 6/15 [08:48<13:09, 87.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.04267892657509564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  47%|████▋     | 7/15 [10:14<11:39, 87.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.03579133368207534\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  53%|█████▎    | 8/15 [11:42<10:11, 87.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.029969054543854373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  60%|██████    | 9/15 [13:09<08:44, 87.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.023491860215711455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  67%|██████▋   | 10/15 [14:36<07:16, 87.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.020394237085527397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  73%|███████▎  | 11/15 [16:04<05:49, 87.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.019622690552894614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  80%|████████  | 12/15 [17:31<04:22, 87.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.015630869830554155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  87%|████████▋ | 13/15 [18:59<02:54, 87.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.015729057546674374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  93%|█████████▎| 14/15 [20:26<01:27, 87.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.01400521895036371\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 100%|██████████| 15/15 [21:53<00:00, 87.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.012883164258949508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = BertForTokenClassification.from_pretrained('bert-base-multilingual-cased', num_labels=len(tag2idx))\n",
    "model.cuda()\n",
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters())\n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "\n",
    "optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)\n",
    "\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "#     print (pred_flat, labels_flat)\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "epochs = 15\n",
    "max_grad_norm = 1.0\n",
    "\n",
    "for _ in trange(epochs, desc=\"Epoch\"):\n",
    "    # TRAIN loop\n",
    "    model.train()\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        # forward pass\n",
    "        loss = model(b_input_ids, token_type_ids=None,\n",
    "                     attention_mask=b_input_mask, labels=b_labels)\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        model.zero_grad()\n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss / nb_tr_steps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), './prefix-finetuning1.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function receives a sentence with its labels, and the tokenized sentence and labels\n",
    "def aggr_toks_labels_tags(orig_words, orig_labels, tok_wordps, tok_labels, predicted_tags):\n",
    "    \n",
    "    joint_tokens = []\n",
    "    joint_labels = []\n",
    "    joint_predicted = []\n",
    "#     joint_test = []\n",
    "    \n",
    "    for word in orig_words:\n",
    "        aggregated_tokenized = \"\"\n",
    "        aggregated_label = \"\"\n",
    "        aggregated_predicted = \"\"\n",
    "        aggregated_test = \"\"\n",
    "        \n",
    "        while aggregated_tokenized != word:\n",
    "#             print(len(tok_sent))\n",
    "            tmpTok = tok_wordps.pop(0)\n",
    "#             print(tmpTok)\n",
    "#             print(joint_tokens)\n",
    "            if tmpTok.startswith(\"##\"):\n",
    "                tmpTok = tmpTok[2:]\n",
    "                \n",
    "            tmpLab = tok_labels.pop(0)\n",
    "#             if aggregated_label == \"\":\n",
    "            aggregated_label += '^'\n",
    "            aggregated_label += tmpLab\n",
    "\n",
    "                \n",
    "            tmpPred = predicted_tags.pop(0)\n",
    "#             print(tmpPred)\n",
    "\n",
    "            aggregated_predicted += '^'\n",
    "            aggregated_predicted += tmpPred\n",
    "#             if aggregated_predicted == \"\":\n",
    "#                 aggregated_predicted = tmpPred\n",
    "                \n",
    "#             tmpTest = test_tags.pop(0)\n",
    "#             if aggregated_test == \"\":\n",
    "#                 aggregated_test = tmpTest\n",
    "                \n",
    "            aggregated_tokenized += tmpTok\n",
    "#             print(aggregated_tokenized)\n",
    "            \n",
    "        joint_tokens.append(aggregated_tokenized)\n",
    "        joint_labels.append(aggregated_label)\n",
    "        joint_predicted.append(aggregated_predicted)\n",
    "#         joint_test.append(aggregated_test)\n",
    "        \n",
    "    assert len(joint_tokens) == len(orig_words)\n",
    "    assert len(joint_tokens) == len(joint_predicted)\n",
    "    return joint_tokens, joint_labels, joint_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "def delete_pads_from_preds(predicted_tags, test_tags):\n",
    "    clean_predicted = []\n",
    "    clean_test = []\n",
    "    \n",
    "    for ix in range(0, len(test_tags)):\n",
    "        if test_tags[ix] != 'PAD':\n",
    "            clean_predicted.append(predicted_tags[ix])\n",
    "            clean_test.append(test_tags[ix])\n",
    "            \n",
    "    return clean_predicted, clean_test\n",
    "    \n",
    "def calculate_accuracy(df):\n",
    "    numOfCorrectPredictions = 0\n",
    "    for index in df.index:\n",
    "        orig_pos = df.at[index, 'test_tag']\n",
    "        pred_pos = df.at[index, 'predicted_tag']\n",
    "        if orig_pos == pred_pos:\n",
    "            numOfCorrectPredictions += 1\n",
    "    return numOfCorrectPredictions/len(df)\n",
    "                \n",
    "def test_model(sentence, labels, tok_sent, tok_labels, corres_tokens, sent_id):\n",
    "    input_ids, tags, attention_masks = pad_sentences_and_labels([tok_sent], [tok_labels])\n",
    "\n",
    "    val_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "    val_tags = torch.tensor(tags, dtype=torch.long)\n",
    "    val_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "    test_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "    test_sampler = SequentialSampler(test_data)\n",
    "    test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=bs)\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions, true_labels = [], []\n",
    "    counter = 0\n",
    "    for batch in test_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(b_input_ids, token_type_ids=None,\n",
    "                                attention_mask=b_input_mask, labels=b_labels)\n",
    "            logits = model(b_input_ids, token_type_ids=None,\n",
    "                         attention_mask=b_input_mask)\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        predictions.append([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        \n",
    "        true_labels.append(label_ids)\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "\n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        nb_eval_examples += b_input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    \n",
    "    pred_tags = [idx2tag[p_ii] for p in predictions for p_i in p for p_ii in p_i]\n",
    "    test_tags = [idx2tag[l_ii] for l in true_labels for l_i in l for l_ii in l_i]\n",
    "#     print(list(zip(pred_tags, test_tags)))\n",
    "    # -----------------------------------------------------------------------\n",
    "    clean_predicted, clean_test = delete_pads_from_preds(pred_tags, test_tags)\n",
    "    joint_tokenized, joint_labels, preds, tests = aggr_toks_labels_tags(sentence, labels, tok_sent, tok_labels, \n",
    "                                                                        clean_predicted, clean_test)\n",
    "    \n",
    "    tmp = {'word': sentence, 'orig_label': labels, 'predicted_tag': preds, 'test_tag': tests, \n",
    "           'corresToken': corres_tokens, 'sent_id': sent_id}\n",
    "    tmp_df = pd.DataFrame(data=tmp)\n",
    "    # -----------------------------------------------------------------------\n",
    "    \n",
    "#     y_true = pd.Series(test_tags)\n",
    "#     y_pred = pd.Series(pred_tags)\n",
    "#     cross_tab = pd.crosstab(y_true, y_pred, rownames=['Real Label'], colnames=['Prediction'], margins=True)\n",
    "#     report = classification_report(y_true, y_pred)\n",
    "#     print(report)\n",
    "#     print(tmp_df)\n",
    "    return tmp_df\n",
    "\n",
    "full_df = pd.DataFrame()\n",
    "dev_tokenized_texts, dev_tokenized_labels = tokenize(dev_sentences, dev_labels)\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(dev_sentences, dev_labels, dev_tokenized_texts, \n",
    "                                                                   dev_tokenized_labels, dev_corresTokens, \n",
    "                                                                   dev_sent_ids):\n",
    "    test_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_df = full_df.append(test_df, ignore_index=True, sort=False)\n",
    "\n",
    "# full_df\n",
    "f1_accuracy = calculate_accuracy(full_df)\n",
    "print(\"Accuracy (F1): = {}\".format(f1_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_df.iloc[0:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename_prefix_output(prefix_df):\n",
    "    prefix_df = prefix_df.rename(columns={\"orig_label\": \"orig_prefix\", \"predicted_tag\": \"predicted_prefix\"})\n",
    "    \n",
    "def rename_host_output(host_df):\n",
    "    host_df = host_df.rename(columns={\"orig_label\": \"orig_host\", \"predicted_tag\": \"predicted_host\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rename_prefix_output(full_df)\n",
    "full_df.to_csv('prefix-5-setting1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "host_df = pd.read_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def join_prefix_host(prefix_df, host_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from more_itertools import unique_everseen\n",
    "\n",
    "def unique_vals_to_list(df):\n",
    "    for index in df.index:\n",
    "        joint_pred = df.at[index, 'predicted_tag']\n",
    "        joint_orig = df.at[index, 'orig_label']\n",
    "        \n",
    "        predicted_tag_list = joint_pred.split('^')\n",
    "        predicted_tag_list_no_empty = list(filter(None, predicted_tag_list))\n",
    "        original_tag_list = joint_orig.split('^')\n",
    "        original_tag_list_no_empty = list(filter(None, original_tag_list))\n",
    "\n",
    "        \n",
    "        df.at[index, 'predicted_tag'] = list(unique_everseen(predicted_tag_list_no_empty))\n",
    "        df.at[index, 'orig_label'] = list(unique_everseen(original_tag_list_no_empty))\n",
    "        \n",
    "        \n",
    "unique_vals_to_list(full_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exact_match_accuracy(df):\n",
    "    exact_matches = 0\n",
    "    for index in df.index:\n",
    "        if df.at[index, 'orig_label'] == df.at[index, 'predicted_tag']:\n",
    "            exact_matches += 1\n",
    "            \n",
    "    return exact_matches\n",
    "\n",
    "print(\"Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_df)/len(full_df) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def existence_accuracy(df):\n",
    "    # correct tag = appeared in predicted and in gold\n",
    "    total_orig_num_of_labels = 0\n",
    "    total_predicted_num_of_labels = 0\n",
    "    total_num_of_correct_tags = 0\n",
    "    \n",
    "    for index in df.index:\n",
    "        orig_list = df.at[index, 'orig_label']\n",
    "        predicted_list = df.at[index, 'predicted_tag']\n",
    "        total_orig_num_of_labels += len(orig_list)\n",
    "        total_predicted_num_of_labels += len(predicted_list)\n",
    "        total_num_of_correct_tags += len(set(orig_list).intersection(set(predicted_list)))\n",
    "        \n",
    "    precision = total_num_of_correct_tags / total_predicted_num_of_labels * 100\n",
    "    recall = total_num_of_correct_tags / total_orig_num_of_labels * 100\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "    print(\"Precision: {0:.2f}%\".format(precision))\n",
    "    print(\"Recall: {0:.2f}%\".format(recall))\n",
    "    print(\"F1: {0:.2f}%\".format(f1))\n",
    "    \n",
    "existence_accuracy(full_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating standard df for multi-label pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class StdDf(object):\n",
    "    def __init__(self):\n",
    "        self.data = pd.concat([train, dev, test])\n",
    "        self.tag_vals = list(set(data['upostag'].values))\n",
    "        self.df = pd.DataFrame(columns = tag_vals)\n",
    "        self.create_multilabel_df()\n",
    "        \n",
    "    def create_multilabel_df(self):        \n",
    "        self.df['sent_id'] = '0'\n",
    "        self.df['token_id'] = '0'\n",
    "        self.df['token'] = ''\n",
    "        cols = self.df.columns.tolist()\n",
    "        cols = cols[-3:] + cols[:-3]\n",
    "        self.df = self.df[cols]\n",
    "        self.df[self.df.columns[3:]] = 0\n",
    "\n",
    "\n",
    "std_df = StdDf()\n",
    "std_df.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def raw_to_multilabel_df(raw_df):\n",
    "    multi_df = StdDf()\n",
    "    for index in raw_df.index:\n",
    "        multi_df.df.at[index, 'sent_id'] = raw_df.at[index, 'sent_id']\n",
    "        multi_df.df.at[index, 'token_id'] = raw_df.at[index, 'token_id']\n",
    "        multi_df.df.at[index, 'token'] = raw_df.at[index, 'token_str']\n",
    "        \n",
    "        l_pos_tags = raw_df.at[index, 'upostag']\n",
    "        multi_df.df.at[index, l_pos_tags] = 1\n",
    "        \n",
    "    return multi_df.df\n",
    "        \n",
    "    \n",
    "multi_dev_df = raw_to_multilabel_df(dev_df)\n",
    "multi_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_dev_df[multi_dev_df.columns[3:]] = 0\n",
    "\n",
    "for index in dev_df.index:\n",
    "    l_pos_tags = dev_df.at[index, 'upostag']\n",
    "    multi_dev_df.at[index, l_pos_tags] = 1\n",
    "    \n",
    "multi_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "morphbert",
   "language": "python",
   "name": "morphbert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}