POS-Host-Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "import bclm\n",
    "\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import classification_report\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertForTokenClassification, BertAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(3)\n",
    "np.random.seed(3)\n",
    "torch.cuda.manual_seed_all(3)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = bclm.read_dataframe('spmrl', subset='train')\n",
    "train_df = bclm.get_token_df(train, ['upostag'])\n",
    "train_df['token_str'] = train_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "dev = bclm.read_dataframe('spmrl', subset='dev')\n",
    "dev_df = bclm.get_token_df(dev, ['upostag'])\n",
    "dev_df['token_str'] = dev_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "test = bclm.read_dataframe('spmrl', subset='test')\n",
    "test_df = bclm.get_token_df(test, ['upostag'])\n",
    "test_df['token_str'] = test_df['token_str'].str.replace('”','\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_df['prefix'] = ''\n",
    "train_df['host'] = ''\n",
    "\n",
    "dev_df['prefix'] = ''\n",
    "dev_df['host'] = ''\n",
    "\n",
    "test_df['prefix'] = ''\n",
    "test_df['host'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>upostag</th>\n",
       "      <th>set</th>\n",
       "      <th>prefix</th>\n",
       "      <th>host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>CDT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>BN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>PREPOSITION^NNP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>לישראל</td>\n",
       "      <td>PREPOSITION^NNP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>כשהם</td>\n",
       "      <td>TEMP^PRP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>נרשמים</td>\n",
       "      <td>BN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>כמתנדבים</td>\n",
       "      <td>PREPOSITION^NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>אך</td>\n",
       "      <td>CC</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>למעשה</td>\n",
       "      <td>RB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>משמשים</td>\n",
       "      <td>BN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>עובדים</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>שכירים</td>\n",
       "      <td>JJ</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>זולים</td>\n",
       "      <td>JJ</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>.</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>תופעה</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>זו</td>\n",
       "      <td>PRP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>התבררה</td>\n",
       "      <td>VB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>אתמול</td>\n",
       "      <td>RB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>בוועדת</td>\n",
       "      <td>PREPOSITION^NNT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>העבודה</td>\n",
       "      <td>DEF^NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>והרווחה</td>\n",
       "      <td>CONJ^DEF^NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>של</td>\n",
       "      <td>POS</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>הכנסת</td>\n",
       "      <td>DEF^NNP</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>שדנה</td>\n",
       "      <td>REL^VB</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>בנושא</td>\n",
       "      <td>PREPOSITION^NNT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>העסקת</td>\n",
       "      <td>NNT</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>עובדים</td>\n",
       "      <td>NN</td>\n",
       "      <td>dev</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sent_id  token_id token_str          upostag  set prefix host\n",
       "0         1         1     עשרות              CDT  dev            \n",
       "1         1         2     אנשים               NN  dev            \n",
       "2         1         3    מגיעים               BN  dev            \n",
       "3         1         4   מתאילנד  PREPOSITION^NNP  dev            \n",
       "4         1         5    לישראל  PREPOSITION^NNP  dev            \n",
       "5         1         6      כשהם         TEMP^PRP  dev            \n",
       "6         1         7    נרשמים               BN  dev            \n",
       "7         1         8  כמתנדבים   PREPOSITION^NN  dev            \n",
       "8         1         9         ,             yyCM  dev            \n",
       "9         1        10        אך               CC  dev            \n",
       "10        1        11     למעשה               RB  dev            \n",
       "11        1        12    משמשים               BN  dev            \n",
       "12        1        13    עובדים               NN  dev            \n",
       "13        1        14    שכירים               JJ  dev            \n",
       "14        1        15     זולים               JJ  dev            \n",
       "15        1        16         .            yyDOT  dev            \n",
       "16        2         1     תופעה               NN  dev            \n",
       "17        2         2        זו              PRP  dev            \n",
       "18        2         3    התבררה               VB  dev            \n",
       "19        2         4     אתמול               RB  dev            \n",
       "20        2         5    בוועדת  PREPOSITION^NNT  dev            \n",
       "21        2         6    העבודה           DEF^NN  dev            \n",
       "22        2         7   והרווחה      CONJ^DEF^NN  dev            \n",
       "23        2         8        של              POS  dev            \n",
       "24        2         9     הכנסת          DEF^NNP  dev            \n",
       "25        2        10         ,             yyCM  dev            \n",
       "26        2        11      שדנה           REL^VB  dev            \n",
       "27        2        12     בנושא  PREPOSITION^NNT  dev            \n",
       "28        2        13     העסקת              NNT  dev            \n",
       "29        2        14    עובדים               NN  dev            "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prefix_corretion(prefix):\n",
    "    not_prefixes = ['VB^POS','VB^DUMMY_AT','VB^AT', 'IN^DUMMY_AT', 'NN', 'IN', 'P',\n",
    "                   'POS', 'RB', 'AT']\n",
    "    first_prefix = ['DEF^BN^AT', 'CONJ^VB^AT','CONJ^BN^AT', 'CONJ^BN^AT', 'REL^VB^AT',\n",
    "                   'PREPOSITION^RB', 'DEF^yyQUOT', 'IN^yyQUOT', 'CONJ^yyQUOT^DEF',\n",
    "                   'PREPOSITION^yyQUOT^PREPOSITION^DEF', 'REL^yyQUOT', 'REL^IN', 'CONJ^IN',\n",
    "                   'PREPOSITION^yyQUOT', 'PREPOSITION^POS', 'PREPOSITION^IN', 'PREPOSITION^yyQUOT^DEF',\n",
    "                   'REL^yyQUOT^PREPOSITION', 'CONJ^yyQUOT', 'REL^AT']\n",
    "    \n",
    "    two_first_prefixes = ['CONJ^PREPOSITION^yyQUOT', 'PREPOSITION^DEF^yyQUOT', 'CONJ^DEF^yyQUOT']\n",
    "    \n",
    "    if prefix in not_prefixes:\n",
    "        host = prefix\n",
    "        prefix = '-'\n",
    "    \n",
    "    elif prefix in first_prefix:\n",
    "        tag_list = prefix.split('^')\n",
    "        prefix = tag_list[0]\n",
    "        host = '^'.join(tag_list[1:])\n",
    "        \n",
    "    elif prefix in two_first_prefixes:\n",
    "        tag_list = prefix.split('^')\n",
    "        prefix = '^'.join(tag_list[0:2])\n",
    "        host = '^'.join(tag_list[2:])\n",
    "        \n",
    "    else:\n",
    "        host = \"\"\n",
    "        prefix = prefix\n",
    "        \n",
    "    return prefix, host\n",
    "        \n",
    "\n",
    "def full_tag_to_prefix_host(df):\n",
    "    for index in df.index:\n",
    "        full_tag = df.at[index, 'upostag']\n",
    "        full_tag_list = full_tag.split('^')\n",
    "        if len(full_tag_list) > 1:\n",
    "            prefix = '^'.join(full_tag_list[:-1])\n",
    "            prefix, host = prefix_corretion(prefix)\n",
    "            df.at[index, 'prefix'] = prefix\n",
    "            if len(host) > 0:\n",
    "                host += '^'\n",
    "            host += full_tag_list[-1]\n",
    "            df.at[index, 'host'] = host\n",
    "        else:\n",
    "            df.at[index, 'prefix'] = '-'\n",
    "            df.at[index, 'host'] = full_tag_list[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>upostag</th>\n",
       "      <th>set</th>\n",
       "      <th>prefix</th>\n",
       "      <th>host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>93474</th>\n",
       "      <td>5436</td>\n",
       "      <td>15</td>\n",
       "      <td>נעליים</td>\n",
       "      <td>NN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93475</th>\n",
       "      <td>5436</td>\n",
       "      <td>16</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93476</th>\n",
       "      <td>5436</td>\n",
       "      <td>17</td>\n",
       "      <td>איך</td>\n",
       "      <td>QW</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>QW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93477</th>\n",
       "      <td>5436</td>\n",
       "      <td>18</td>\n",
       "      <td>מנהלים</td>\n",
       "      <td>BN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>BN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93478</th>\n",
       "      <td>5436</td>\n",
       "      <td>19</td>\n",
       "      <td>חשבון</td>\n",
       "      <td>NN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93479</th>\n",
       "      <td>5436</td>\n",
       "      <td>20</td>\n",
       "      <td>בבנק</td>\n",
       "      <td>PREPOSITION^DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>PREPOSITION^DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93480</th>\n",
       "      <td>5436</td>\n",
       "      <td>21</td>\n",
       "      <td>.</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyDOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93481</th>\n",
       "      <td>5437</td>\n",
       "      <td>1</td>\n",
       "      <td>אילנה</td>\n",
       "      <td>NNP</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93482</th>\n",
       "      <td>5437</td>\n",
       "      <td>2</td>\n",
       "      <td>נחום</td>\n",
       "      <td>NNP</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93483</th>\n",
       "      <td>5437</td>\n",
       "      <td>3</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93484</th>\n",
       "      <td>5437</td>\n",
       "      <td>4</td>\n",
       "      <td>מזכירת</td>\n",
       "      <td>NNT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93485</th>\n",
       "      <td>5437</td>\n",
       "      <td>5</td>\n",
       "      <td>הקיבוץ</td>\n",
       "      <td>DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93486</th>\n",
       "      <td>5437</td>\n",
       "      <td>6</td>\n",
       "      <td>,</td>\n",
       "      <td>yyCM</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93487</th>\n",
       "      <td>5437</td>\n",
       "      <td>7</td>\n",
       "      <td>נתנה</td>\n",
       "      <td>VB</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>VB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93488</th>\n",
       "      <td>5437</td>\n",
       "      <td>8</td>\n",
       "      <td>ביטוי</td>\n",
       "      <td>NN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93489</th>\n",
       "      <td>5437</td>\n",
       "      <td>9</td>\n",
       "      <td>לחרדות</td>\n",
       "      <td>PREPOSITION^DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>PREPOSITION^DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93490</th>\n",
       "      <td>5437</td>\n",
       "      <td>10</td>\n",
       "      <td>של</td>\n",
       "      <td>POS</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>POS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93491</th>\n",
       "      <td>5437</td>\n",
       "      <td>11</td>\n",
       "      <td>בני</td>\n",
       "      <td>NNT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>NNT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93492</th>\n",
       "      <td>5437</td>\n",
       "      <td>12</td>\n",
       "      <td>הקיבוץ</td>\n",
       "      <td>DEF^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>DEF</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93493</th>\n",
       "      <td>5437</td>\n",
       "      <td>13</td>\n",
       "      <td>:</td>\n",
       "      <td>yyCLN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyCLN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93494</th>\n",
       "      <td>5437</td>\n",
       "      <td>14</td>\n",
       "      <td>\"</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyQUOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93495</th>\n",
       "      <td>5437</td>\n",
       "      <td>15</td>\n",
       "      <td>זה</td>\n",
       "      <td>PRP</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>PRP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93496</th>\n",
       "      <td>5437</td>\n",
       "      <td>16</td>\n",
       "      <td>רע</td>\n",
       "      <td>JJ</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>JJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93497</th>\n",
       "      <td>5437</td>\n",
       "      <td>17</td>\n",
       "      <td>מאוד</td>\n",
       "      <td>RB</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>RB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93498</th>\n",
       "      <td>5437</td>\n",
       "      <td>18</td>\n",
       "      <td>אם</td>\n",
       "      <td>CC</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>CC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93499</th>\n",
       "      <td>5437</td>\n",
       "      <td>19</td>\n",
       "      <td>יגיעו</td>\n",
       "      <td>VB</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>VB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93500</th>\n",
       "      <td>5437</td>\n",
       "      <td>20</td>\n",
       "      <td>אלינו</td>\n",
       "      <td>IN^S_PRN</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>IN^S_PRN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93501</th>\n",
       "      <td>5437</td>\n",
       "      <td>21</td>\n",
       "      <td>מכורח</td>\n",
       "      <td>PREPOSITION^NN</td>\n",
       "      <td>train</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93502</th>\n",
       "      <td>5437</td>\n",
       "      <td>22</td>\n",
       "      <td>\"</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyQUOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93503</th>\n",
       "      <td>5437</td>\n",
       "      <td>23</td>\n",
       "      <td>.</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>train</td>\n",
       "      <td>-</td>\n",
       "      <td>yyDOT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sent_id  token_id token_str             upostag    set  \\\n",
       "93474     5436        15    נעליים                  NN  train   \n",
       "93475     5436        16         ,                yyCM  train   \n",
       "93476     5436        17       איך                  QW  train   \n",
       "93477     5436        18    מנהלים                  BN  train   \n",
       "93478     5436        19     חשבון                  NN  train   \n",
       "93479     5436        20      בבנק  PREPOSITION^DEF^NN  train   \n",
       "93480     5436        21         .               yyDOT  train   \n",
       "93481     5437         1     אילנה                 NNP  train   \n",
       "93482     5437         2      נחום                 NNP  train   \n",
       "93483     5437         3         ,                yyCM  train   \n",
       "93484     5437         4    מזכירת                 NNT  train   \n",
       "93485     5437         5    הקיבוץ              DEF^NN  train   \n",
       "93486     5437         6         ,                yyCM  train   \n",
       "93487     5437         7      נתנה                  VB  train   \n",
       "93488     5437         8     ביטוי                  NN  train   \n",
       "93489     5437         9    לחרדות  PREPOSITION^DEF^NN  train   \n",
       "93490     5437        10        של                 POS  train   \n",
       "93491     5437        11       בני                 NNT  train   \n",
       "93492     5437        12    הקיבוץ              DEF^NN  train   \n",
       "93493     5437        13         :               yyCLN  train   \n",
       "93494     5437        14         \"              yyQUOT  train   \n",
       "93495     5437        15        זה                 PRP  train   \n",
       "93496     5437        16        רע                  JJ  train   \n",
       "93497     5437        17      מאוד                  RB  train   \n",
       "93498     5437        18        אם                  CC  train   \n",
       "93499     5437        19     יגיעו                  VB  train   \n",
       "93500     5437        20     אלינו            IN^S_PRN  train   \n",
       "93501     5437        21     מכורח      PREPOSITION^NN  train   \n",
       "93502     5437        22         \"              yyQUOT  train   \n",
       "93503     5437        23         .               yyDOT  train   \n",
       "\n",
       "                prefix      host  \n",
       "93474                -        NN  \n",
       "93475                -      yyCM  \n",
       "93476                -        QW  \n",
       "93477                -        BN  \n",
       "93478                -        NN  \n",
       "93479  PREPOSITION^DEF        NN  \n",
       "93480                -     yyDOT  \n",
       "93481                -       NNP  \n",
       "93482                -       NNP  \n",
       "93483                -      yyCM  \n",
       "93484                -       NNT  \n",
       "93485              DEF        NN  \n",
       "93486                -      yyCM  \n",
       "93487                -        VB  \n",
       "93488                -        NN  \n",
       "93489  PREPOSITION^DEF        NN  \n",
       "93490                -       POS  \n",
       "93491                -       NNT  \n",
       "93492              DEF        NN  \n",
       "93493                -     yyCLN  \n",
       "93494                -    yyQUOT  \n",
       "93495                -       PRP  \n",
       "93496                -        JJ  \n",
       "93497                -        RB  \n",
       "93498                -        CC  \n",
       "93499                -        VB  \n",
       "93500                -  IN^S_PRN  \n",
       "93501      PREPOSITION        NN  \n",
       "93502                -    yyQUOT  \n",
       "93503                -     yyDOT  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_tag_to_prefix_host(train_df)\n",
    "full_tag_to_prefix_host(dev_df)\n",
    "full_tag_to_prefix_host(test_df)\n",
    "\n",
    "train_df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train2 = train[train['xpostag'] != train['upostag']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train2['xpostag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train2[train2['xpostag'] == 'S_PRN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev2 = dev[dev['xpostag'] != dev['upostag']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev2['xpostag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(dev2)/len(dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'PAD': 0, 'IN^??': 1, 'NN^yyDOT': 2, 'VB^AT^S_PRN': 3, 'yyQUOT^NNP': 4, 'IN': 5, 'CONJ': 6, 'MD': 7, 'yyQM': 8, 'yyCM': 9, 'yyCLN': 10, 'P^NN': 11, 'AT': 12, 'yyQUOT^NN': 13, 'yyQUOT^PREPOSITION^NN': 14, 'NN': 15, 'INTJ': 16, 'RB': 17, 'yySCLN': 18, 'yyQUOT^CDT': 19, 'VB^DUMMY_AT^S_ANP': 20, 'REL': 21, 'IN^IN': 22, 'yyQUOT^DEF^JJ': 23, 'IN^JJT': 24, 'POS': 25, 'AT^S_PRN': 26, 'EX': 27, 'IN^DUMMY_AT^PRP': 28, 'BNT': 29, 'IN^NNT': 30, 'yyQUOT^VB': 31, 'POS^S_PRN': 32, 'RB^S_PRN': 33, 'DT': 34, 'BN': 35, '??': 36, 'DEF': 37, 'NNP': 38, 'PREPOSITION': 39, 'IN^NNP': 40, 'VB^AT^PRP': 41, 'JJ': 42, 'yyLRB': 43, 'yyELPS': 44, 'yyEXCL': 45, 'ZVL': 46, 'CD': 47, 'yyDASH': 48, 'CDT': 49, 'yyQUOT^PRP': 50, 'VB^POS^S_PRN': 51, 'yyQUOT^RB': 52, 'BN^AT^PRP': 53, 'NNT': 54, 'yyQUOT^DEF^NN': 55, 'yyQUOT^BN': 56, 'VB': 57, 'AT^PRP': 58, 'NEG': 59, 'yyQUOT^PREPOSITION^DEF^NN': 60, 'yyQUOT^JJ': 61, 'COP': 62, 'yyQUOT^IN': 63, 'VB^DUMMY_AT^PRP': 64, 'IN^NCD': 65, 'yyDOT': 66, 'DTT': 67, 'IN^NN': 68, 'PRP': 69, 'yyQUOT^COP': 70, 'JJT': 71, 'IN^S_PRN': 72, 'yyQUOT': 73, 'IN^RB': 74, 'CC': 75, 'IN^CDT': 76, 'QW': 77, 'IN^VB': 78, 'NCD': 79, 'yyQUOT^MD': 80, 'VB^AT^S_ANP': 81, 'P': 82, 'yyRRB': 83, 'IN^DTT': 84, 'yyQUOT^NNT': 85, 'IN^PRP': 86}\n",
      "87\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([train_df, dev_df, test_df])\n",
    "data.head()\n",
    "tag_vals = list(set(data[\"host\"].values))\n",
    "tags = ['PAD'] + tag_vals\n",
    "tag2idx = {tag:idx for idx, tag in enumerate(tags)}\n",
    "idx2tag = {idx:tag for idx, tag in enumerate(tags)}\n",
    "\n",
    "print(tag2idx)\n",
    "# print(idx2tag)\n",
    "print(len(tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for label in tag_vals:\n",
    "#     print(data[data['prefix'] == 'PREPOSITIONIN'])\n",
    "    \n",
    "data[data['host'] == 'IN^DUMMY_AT^PRP']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['הם', 'התבקשו', 'לדווח', 'למשטרה', 'על', 'תנועותיהם', '.']\n",
      "['PRP', 'VB', 'VB', 'NN', 'IN', 'NN', 'yyDOT']\n"
     ]
    }
   ],
   "source": [
    "class sentenceGetter(object):\n",
    "    def __init__(self, data, max_sent=None):\n",
    "        self.index = 0\n",
    "        self.max_sent = max_sent\n",
    "        self.tokens = data['token_str']\n",
    "        self.labels = data['host']\n",
    "        #for evaluating by word-accuracy\n",
    "        self.correspondingToken = data['token_id']\n",
    "        self.orig_sent_id = data['sent_id']\n",
    "    \n",
    "    def sentences(self):\n",
    "        sent = []\n",
    "        counter = 0\n",
    "        \n",
    "        for token,label, corres_tok, sent_id in zip(self.tokens, self.labels, self.correspondingToken, self.orig_sent_id):\n",
    "            sent.append((token, label, corres_tok, sent_id))\n",
    "            if token.strip() == \".\":\n",
    "                yield sent\n",
    "                sent = []\n",
    "                counter += 1\n",
    "            if self.max_sent is not None and counter >= self.max_sent:\n",
    "                return\n",
    "\n",
    "train_getter = sentenceGetter(train_df)\n",
    "dev_getter = sentenceGetter(dev_df)\n",
    "test_getter = sentenceGetter(test_df)\n",
    "\n",
    "train_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "train_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "\n",
    "# dev_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "# dev_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "# dev_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "# dev_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "\n",
    "# test_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "# test_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "\n",
    "print(train_sentences[10])\n",
    "print(train_labels[10])\n",
    "\n",
    "# print(len(dev_sentences))\n",
    "\n",
    "# del dev_sentences[296]\n",
    "# del dev_labels[296]\n",
    "# del dev_corresTokens[296]\n",
    "# del dev_sent_ids[296]\n",
    "\n",
    "\n",
    "# del dev_sentences[226]\n",
    "# del dev_labels[226]\n",
    "# del dev_corresTokens[226]\n",
    "# del dev_sent_ids[226]\n",
    "\n",
    "\n",
    "# del dev_sentences[57]\n",
    "# del dev_labels[57]\n",
    "# del dev_corresTokens[57]\n",
    "# del dev_sent_ids[57]\n",
    "\n",
    "\n",
    "# del dev_sentences[49]\n",
    "# del dev_labels[49]\n",
    "# del dev_corresTokens[49]\n",
    "# del dev_sent_ids[49]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda\n",
      "Number of gpus: 4\n",
      "Name of gpu: GeForce GTX 1080 Ti\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.set_device(2)\n",
    "\n",
    "print(\"Device: \" + str(device))\n",
    "print(\"Number of gpus: \" + str(n_gpu))\n",
    "print(\"Name of gpu: \" + torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 150\n",
    "bs = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The pre-trained model you are loading is a cased model but you have not set `do_lower_case` to False. We are setting `do_lower_case=False` for you but you may want to check this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['הם', 'ה', '##ת', '##בק', '##שו', 'ל', '##דו', '##וח', 'ל', '##משטרה', 'על', 'ת', '##נוע', '##ות', '##יהם', '.']\n",
      "['PRP', 'VB', 'VB', 'VB', 'VB', 'VB', 'VB', 'VB', 'NN', 'NN', 'IN', 'NN', 'NN', 'NN', 'NN', 'yyDOT']\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
    "def tokenize(sentences, orig_labels):\n",
    "    tokenized_texts = []\n",
    "    labels = []\n",
    "    for sent, sent_labels in zip(sentences, orig_labels):\n",
    "        bert_tokens = []\n",
    "        bert_labels = []\n",
    "        for orig_token, orig_label in zip(sent, sent_labels):\n",
    "            b_tokens = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(b_tokens)\n",
    "            for b_token in b_tokens:\n",
    "                bert_labels.append(orig_label)\n",
    "        tokenized_texts.append(bert_tokens)\n",
    "        labels.append(bert_labels)\n",
    "        assert len(bert_tokens) == len(bert_labels)\n",
    "    return tokenized_texts, labels\n",
    "\n",
    "train_tokenized_texts, train_tokenized_labels = tokenize(train_sentences, train_labels)\n",
    "print(train_tokenized_texts[10])\n",
    "print(train_tokenized_labels[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentences_and_labels(tokenized_texts, labels):\n",
    "    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                              maxlen = MAX_LEN, dtype = \"float32\", truncating = \"post\", padding = \"post\", value = tag2idx['PAD'])\n",
    "    tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels], \n",
    "                         maxlen = MAX_LEN, value = tag2idx['PAD'], padding = \"post\",\n",
    "                        dtype = \"float32\", truncating = \"post\")\n",
    "    attention_masks = [[float(i>0) for i in ii] for ii in input_ids]\n",
    "    return input_ids, tags, attention_masks\n",
    "\n",
    "input_ids, tags, attention_masks = pad_sentences_and_labels(train_tokenized_texts, train_tokenized_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "tr_tags = torch.tensor(tags, dtype=torch.long)\n",
    "tr_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "# train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, batch_size=bs, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Error(s) in loading state_dict for BertForTokenClassification:\n\tsize mismatch for classifier.weight: copying a param with shape torch.Size([48, 768]) from checkpoint, the shape in current model is torch.Size([87, 768]).\n\tsize mismatch for classifier.bias: copying a param with shape torch.Size([48]) from checkpoint, the shape in current model is torch.Size([87]).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-eb9c100b7c01>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmodel_state_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'prefix-finetuning1.pt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m model = BertForTokenClassification.from_pretrained('bert-base-multilingual-cased', num_labels=len(tag2idx),\n\u001b[0;32m----> 3\u001b[0;31m                                                   state_dict=model_state_dict)\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mFULL_FINETUNING\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/morphbert/lib/python3.7/site-packages/pytorch_pretrained_bert/modeling.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m    652\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_msgs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    653\u001b[0m             raise RuntimeError('Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n\u001b[0;32m--> 654\u001b[0;31m                                model.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n\u001b[0m\u001b[1;32m    655\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    656\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for BertForTokenClassification:\n\tsize mismatch for classifier.weight: copying a param with shape torch.Size([48, 768]) from checkpoint, the shape in current model is torch.Size([87, 768]).\n\tsize mismatch for classifier.bias: copying a param with shape torch.Size([48]) from checkpoint, the shape in current model is torch.Size([87])."
     ]
    }
   ],
   "source": [
    "model = BertForTokenClassification.from_pretrained('bert-base-multilingual-cased', num_labels=len(tag2idx))\n",
    "model.cuda()\n",
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters())\n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "\n",
    "optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)\n",
    "\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "#     print (pred_flat, labels_flat)\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "epochs = 15\n",
    "max_grad_norm = 1.0\n",
    "\n",
    "for _ in trange(epochs, desc=\"Epoch\"):\n",
    "    # TRAIN loop\n",
    "    model.train()\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        # forward pass\n",
    "        loss = model(b_input_ids, token_type_ids=None,\n",
    "                     attention_mask=b_input_mask, labels=b_labels)\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        model.zero_grad()\n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss / nb_tr_steps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function receives a sentence with its labels, and the tokenized sentence and labels\n",
    "def aggr_toks_labels_tags(orig_words, orig_labels, tok_wordps, tok_labels, predicted_tags):\n",
    "    \n",
    "    joint_tokens = []\n",
    "    joint_labels = []\n",
    "    joint_predicted = []\n",
    "#     joint_test = []\n",
    "    \n",
    "    for word in orig_words:\n",
    "        aggregated_tokenized = \"\"\n",
    "        aggregated_label = \"\"\n",
    "        aggregated_predicted = \"\"\n",
    "        aggregated_test = \"\"\n",
    "        \n",
    "        while aggregated_tokenized != word:\n",
    "#             print(len(tok_sent))\n",
    "            tmpTok = tok_wordps.pop(0)\n",
    "#             print(tmpTok)\n",
    "#             print(joint_tokens)\n",
    "            if tmpTok.startswith(\"##\"):\n",
    "                tmpTok = tmpTok[2:]\n",
    "                \n",
    "            tmpLab = tok_labels.pop(0)\n",
    "#             if aggregated_label == \"\":\n",
    "            aggregated_label += '^'\n",
    "            aggregated_label += tmpLab\n",
    "\n",
    "                \n",
    "            tmpPred = predicted_tags.pop(0)\n",
    "#             print(tmpPred)\n",
    "\n",
    "            aggregated_predicted += '^'\n",
    "            aggregated_predicted += tmpPred\n",
    "#             if aggregated_predicted == \"\":\n",
    "#                 aggregated_predicted = tmpPred\n",
    "                \n",
    "#             tmpTest = test_tags.pop(0)\n",
    "#             if aggregated_test == \"\":\n",
    "#                 aggregated_test = tmpTest\n",
    "                \n",
    "            aggregated_tokenized += tmpTok\n",
    "#             print(aggregated_tokenized)\n",
    "            \n",
    "        joint_tokens.append(aggregated_tokenized)\n",
    "        joint_labels.append(aggregated_label)\n",
    "        joint_predicted.append(aggregated_predicted)\n",
    "#         joint_test.append(aggregated_test)\n",
    "        \n",
    "    assert len(joint_tokens) == len(orig_words)\n",
    "    assert len(joint_tokens) == len(joint_predicted)\n",
    "    return joint_tokens, joint_labels, joint_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "def delete_pads_from_preds(predicted_tags, test_tags):\n",
    "    clean_predicted = []\n",
    "    clean_test = []\n",
    "    \n",
    "    for ix in range(0, len(test_tags)):\n",
    "        if test_tags[ix] != 'PAD':\n",
    "            clean_predicted.append(predicted_tags[ix])\n",
    "            clean_test.append(test_tags[ix])\n",
    "            \n",
    "    return clean_predicted, clean_test\n",
    "    \n",
    "def calculate_accuracy(df):\n",
    "    numOfCorrectPredictions = 0\n",
    "    for index in df.index:\n",
    "        orig_pos = df.at[index, 'test_tag']\n",
    "        pred_pos = df.at[index, 'predicted_tag']\n",
    "        if orig_pos == pred_pos:\n",
    "            numOfCorrectPredictions += 1\n",
    "    return numOfCorrectPredictions/len(df)\n",
    "                \n",
    "def test_model(sentence, labels, tok_sent, tok_labels, corres_tokens, sent_id):\n",
    "    input_ids, tags, attention_masks = pad_sentences_and_labels([tok_sent], [tok_labels])\n",
    "\n",
    "    val_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "    val_tags = torch.tensor(tags, dtype=torch.long)\n",
    "    val_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "    test_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "    test_sampler = SequentialSampler(test_data)\n",
    "    test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=bs)\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions, true_labels = [], []\n",
    "    counter = 0\n",
    "    for batch in test_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(b_input_ids, token_type_ids=None,\n",
    "                                attention_mask=b_input_mask, labels=b_labels)\n",
    "            logits = model(b_input_ids, token_type_ids=None,\n",
    "                         attention_mask=b_input_mask)\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        predictions.append([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        \n",
    "        true_labels.append(label_ids)\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "\n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        nb_eval_examples += b_input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    \n",
    "    pred_tags = [idx2tag[p_ii] for p in predictions for p_i in p for p_ii in p_i]\n",
    "    test_tags = [idx2tag[l_ii] for l in true_labels for l_i in l for l_ii in l_i]\n",
    "#     print(list(zip(pred_tags, test_tags)))\n",
    "    # -----------------------------------------------------------------------\n",
    "    clean_predicted, clean_test = delete_pads_from_preds(pred_tags, test_tags)\n",
    "    joint_tokenized, joint_labels, preds, tests = aggr_toks_labels_tags(sentence, labels, tok_sent, tok_labels, \n",
    "                                                                        clean_predicted, clean_test)\n",
    "    \n",
    "    tmp = {'word': sentence, 'orig_label': labels, 'predicted_tag': preds, 'test_tag': tests, \n",
    "           'corresToken': corres_tokens, 'sent_id': sent_id}\n",
    "    tmp_df = pd.DataFrame(data=tmp)\n",
    "    # -----------------------------------------------------------------------\n",
    "    \n",
    "#     y_true = pd.Series(test_tags)\n",
    "#     y_pred = pd.Series(pred_tags)\n",
    "#     cross_tab = pd.crosstab(y_true, y_pred, rownames=['Real Label'], colnames=['Prediction'], margins=True)\n",
    "#     report = classification_report(y_true, y_pred)\n",
    "#     print(report)\n",
    "#     print(tmp_df)\n",
    "    return tmp_df\n",
    "\n",
    "full_df = pd.DataFrame()\n",
    "dev_tokenized_texts, dev_tokenized_labels = tokenize(dev_sentences, dev_labels)\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(dev_sentences, dev_labels, dev_tokenized_texts, \n",
    "                                                                   dev_tokenized_labels, dev_corresTokens, \n",
    "                                                                   dev_sent_ids):\n",
    "    test_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_df = full_df.append(test_df, ignore_index=True, sort=False)\n",
    "\n",
    "# full_df\n",
    "f1_accuracy = calculate_accuracy(full_df)\n",
    "print(\"Accuracy (F1): = {}\".format(f1_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_df.iloc[0:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename_prefix_output(prefix_df):\n",
    "    prefix_df.rename(columns={\"orig_label\": \"orig_prefix\", \"predicted_tag\": \"predicted_prefix\"})\n",
    "    \n",
    "def rename_host_output(host_df):\n",
    "    host_df.rename(columns={\"orig_label\": \"orig_host\", \"predicted_tag\": \"predicted_host\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rename_prefix_output(full_df)\n",
    "full_df.to_csv('host-5-setting1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "host_df = pd.read_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def join_prefix_host(prefix_df, host_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from more_itertools import unique_everseen\n",
    "\n",
    "def unique_vals_to_list(df):\n",
    "    for index in df.index:\n",
    "        joint_pred = df.at[index, 'predicted_tag']\n",
    "        joint_orig = df.at[index, 'orig_label']\n",
    "        \n",
    "        predicted_tag_list = joint_pred.split('^')\n",
    "        predicted_tag_list_no_empty = list(filter(None, predicted_tag_list))\n",
    "        original_tag_list = joint_orig.split('^')\n",
    "        original_tag_list_no_empty = list(filter(None, original_tag_list))\n",
    "\n",
    "        \n",
    "        df.at[index, 'predicted_tag'] = list(unique_everseen(predicted_tag_list_no_empty))\n",
    "        df.at[index, 'orig_label'] = list(unique_everseen(original_tag_list_no_empty))\n",
    "        \n",
    "        \n",
    "unique_vals_to_list(full_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exact_match_accuracy(df):\n",
    "    exact_matches = 0\n",
    "    for index in df.index:\n",
    "        if df.at[index, 'orig_label'] == df.at[index, 'predicted_tag']:\n",
    "            exact_matches += 1\n",
    "            \n",
    "    return exact_matches\n",
    "\n",
    "print(\"Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_df)/len(full_df) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def existence_accuracy(df):\n",
    "    # correct tag = appeared in predicted and in gold\n",
    "    total_orig_num_of_labels = 0\n",
    "    total_predicted_num_of_labels = 0\n",
    "    total_num_of_correct_tags = 0\n",
    "    \n",
    "    for index in df.index:\n",
    "        orig_list = df.at[index, 'orig_label']\n",
    "        predicted_list = df.at[index, 'predicted_tag']\n",
    "        total_orig_num_of_labels += len(orig_list)\n",
    "        total_predicted_num_of_labels += len(predicted_list)\n",
    "        total_num_of_correct_tags += len(set(orig_list).intersection(set(predicted_list)))\n",
    "        \n",
    "    precision = total_num_of_correct_tags / total_predicted_num_of_labels * 100\n",
    "    recall = total_num_of_correct_tags / total_orig_num_of_labels * 100\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "    print(\"Precision: {0:.2f}%\".format(precision))\n",
    "    print(\"Recall: {0:.2f}%\".format(recall))\n",
    "    print(\"F1: {0:.2f}%\".format(f1))\n",
    "    \n",
    "existence_accuracy(full_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating standard df for multi-label pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class StdDf(object):\n",
    "    def __init__(self):\n",
    "        self.data = pd.concat([train, dev, test])\n",
    "        self.tag_vals = list(set(data['upostag'].values))\n",
    "        self.df = pd.DataFrame(columns = tag_vals)\n",
    "        self.create_multilabel_df()\n",
    "        \n",
    "    def create_multilabel_df(self):        \n",
    "        self.df['sent_id'] = '0'\n",
    "        self.df['token_id'] = '0'\n",
    "        self.df['token'] = ''\n",
    "        cols = self.df.columns.tolist()\n",
    "        cols = cols[-3:] + cols[:-3]\n",
    "        self.df = self.df[cols]\n",
    "        self.df[self.df.columns[3:]] = 0\n",
    "\n",
    "\n",
    "std_df = StdDf()\n",
    "std_df.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def raw_to_multilabel_df(raw_df):\n",
    "    multi_df = StdDf()\n",
    "    for index in raw_df.index:\n",
    "        multi_df.df.at[index, 'sent_id'] = raw_df.at[index, 'sent_id']\n",
    "        multi_df.df.at[index, 'token_id'] = raw_df.at[index, 'token_id']\n",
    "        multi_df.df.at[index, 'token'] = raw_df.at[index, 'token_str']\n",
    "        \n",
    "        l_pos_tags = raw_df.at[index, 'upostag']\n",
    "        multi_df.df.at[index, l_pos_tags] = 1\n",
    "        \n",
    "    return multi_df.df\n",
    "        \n",
    "    \n",
    "multi_dev_df = raw_to_multilabel_df(dev_df)\n",
    "multi_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_dev_df[multi_dev_df.columns[3:]] = 0\n",
    "\n",
    "for index in dev_df.index:\n",
    "    l_pos_tags = dev_df.at[index, 'upostag']\n",
    "    multi_dev_df.at[index, l_pos_tags] = 1\n",
    "    \n",
    "multi_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "morphbert",
   "language": "python",
   "name": "morphbert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}