camembert_gpu.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The code and data are adapted from:  https://medium.com/@vitalshchutski/french-nlp-entamez-le-camembert-avec-les-librairies-fast-bert-et-transformers-14e65f84c148"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !conda install torch\n",
    "# !pip install fast-bert==1.9.1\n",
    "# !mkdir model\n",
    "# !mkdir finetuned_model\n",
    "# manully download & install apex => https://github.com/NVIDIA/apex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from fast_bert.data_cls import BertDataBunch \n",
    "from fast_bert.learner_cls import BertLearner\n",
    "from fast_bert.data_lm import BertLMDataBunch\n",
    "from fast_bert.learner_lm import BertLMLearner\n",
    "from fast_bert.metrics import fbeta, roc_auc\n",
    "from fast_bert.prediction import BertClassificationPredictor\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import logging\n",
    "\n",
    "\n",
    "device_cuda = torch.device(\"cuda\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fixing shitty windows encodings => failed "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'utf-8'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.getdefaultencoding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'UTF-8'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import locale; \n",
    "if locale.getpreferredencoding().upper() != 'UTF-8': \n",
    "    locale.setlocale(locale.LC_ALL, ('fr', 'utf-8'))\n",
    "locale.getpreferredencoding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'utf-8'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.getfilesystemencoding()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path('./data/')\n",
    "LOG_PATH = Path('./logs/')\n",
    "MODEL_PATH = Path('./model/')\n",
    "LABEL_PATH = Path('./labels/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create logger\n",
    "logfile = str('logfile.txt')\n",
    "\n",
    "logging.basicConfig(\n",
    "    level=logging.INFO,  #CRITICAL ERROR WARNING  INFO    DEBUG    NOTSET\n",
    "    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "    datefmt='%m/%d/%Y %H:%M:%S',\n",
    "    handlers=[\n",
    "        logging.FileHandler(logfile, 'w', 'utf-8'),\n",
    "        logging.StreamHandler(sys.stdout)\n",
    "    ])\n",
    "\n",
    "logger = logging.getLogger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/AA_xml_manul - 0-200_0-403.csv',encoding = 'utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove NaN\n",
    "df = df[~df['Disease'].isnull()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "df['Disease'] = df['Disease'].apply(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>report_text</th>\n",
       "      <th>source_name</th>\n",
       "      <th>Bioagressor</th>\n",
       "      <th>Disease</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>doryphore des larves de doryphores ont été obs...</td>\n",
       "      <td>AA_PDT_Ile_de_France_2006_005</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>puceronson note l'apparition des premiers puce...</td>\n",
       "      <td>AA_PDT_Ile_de_France_2006_005</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>détermination du seuil de traitement :réalisez...</td>\n",
       "      <td>AA_PDT_Ile_de_France_2006_005</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>protection du maïs semence édité avec la colla...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>situation il est possible d'observer les flétr...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>préconisation toute intervention est aujourd'h...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>situation - prévision le vol est réalisé à plu...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>préconisationune intervention ne se justifiera...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>situation - prévisiondes colonies de metopolop...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>préconisationaucune intervention n'est nécessa...</td>\n",
       "      <td>AA_GC_Midi_Pyrenees_1993_013</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           report_text  \\\n",
       "392  doryphore des larves de doryphores ont été obs...   \n",
       "393  puceronson note l'apparition des premiers puce...   \n",
       "394  détermination du seuil de traitement :réalisez...   \n",
       "395  protection du maïs semence édité avec la colla...   \n",
       "396  situation il est possible d'observer les flétr...   \n",
       "397  préconisation toute intervention est aujourd'h...   \n",
       "398  situation - prévision le vol est réalisé à plu...   \n",
       "399  préconisationune intervention ne se justifiera...   \n",
       "400  situation - prévisiondes colonies de metopolop...   \n",
       "401  préconisationaucune intervention n'est nécessa...   \n",
       "\n",
       "                       source_name  Bioagressor  Disease  \n",
       "392  AA_PDT_Ile_de_France_2006_005            1      0.0  \n",
       "393  AA_PDT_Ile_de_France_2006_005            0      0.0  \n",
       "394  AA_PDT_Ile_de_France_2006_005            0      0.0  \n",
       "395   AA_GC_Midi_Pyrenees_1993_013            0      0.0  \n",
       "396   AA_GC_Midi_Pyrenees_1993_013            1      0.0  \n",
       "397   AA_GC_Midi_Pyrenees_1993_013            0      0.0  \n",
       "398   AA_GC_Midi_Pyrenees_1993_013            0      0.0  \n",
       "399   AA_GC_Midi_Pyrenees_1993_013            0      0.0  \n",
       "400   AA_GC_Midi_Pyrenees_1993_013            0      0.0  \n",
       "401   AA_GC_Midi_Pyrenees_1993_013            0      0.0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nombre de commentaires dans le val_set: 80\n",
      "Nombre de commentaires dans le train_set: 321\n"
     ]
    }
   ],
   "source": [
    "val_set = df.sample(frac=0.2, replace=False, random_state=42)\n",
    "train_set = df.drop(index = val_set.index)\n",
    "print('Nombre de commentaires dans le val_set:',len(val_set))\n",
    "print('Nombre de commentaires dans le train_set:', len(train_set))\n",
    "val_set.to_csv('./data/val_set.csv',encoding = 'utf-8')\n",
    "train_set.to_csv('./data/train_set.csv',encoding = 'utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = df.columns[2:4].to_list() \n",
    "with open('./labels/labels.txt', 'w',encoding = 'utf-8') as f:\n",
    "    for i in labels:\n",
    "        f.write(i + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nombre de bloc de texte: 2801\n"
     ]
    }
   ],
   "source": [
    "#df_texts = pd.read_csv('./data/bsv_chunk256_raw_1001-1200.csv',encoding = 'utf-8')\n",
    "#all_texts = df_texts['report_text'].to_list()\n",
    "#all_texts = [str(t.encode('utf-8')) for t in all_texts] #that's the key to make it work under windows! => mais c'est diable\n",
    "#print('Nombre de bloc de texte:', len(all_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_texts = pd.read_csv('./data/raw_xml_bsv_0-200.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import re\n",
    "\n",
    "# make all elements string\n",
    "df_texts['report_text'] = df_texts['report_text'].astype(str)\n",
    "# Remove null fields\n",
    "df_texts['report_text'] = df_texts['report_text'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))\n",
    "# Make all text lowercase\n",
    "df_texts['report_text'] = df_texts['report_text'].apply(lambda x: x.lower())\n",
    "# Delete stop-words => to be tesred later\n",
    "#stopwords = nltk.corpus.stopwords.words('french')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nombre de bloc de texte: 2297\n"
     ]
    }
   ],
   "source": [
    "all_texts = df_texts['report_text'].to_list()[500:]\n",
    "print('Nombre de bloc de texte:', len(all_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pucerons cendresintervenir avec aphicide nécessaire plus colonies',\n",
       " 'betteraves',\n",
       " 'pucerons jaunisse',\n",
       " 'situationles premiers pucerons sont capturés bacs jaunes.',\n",
       " \"préconisationles betteraves sont sensibles stade feuilles jusqu'à couverture sol. premier traitement sera effectuer prochain réchauffement particulièrement parcelles n'ayant reçu microgranulés semis ayant atteint stade feuilles. utiliser spécialité base pyréthrinoïde.\",\n",
       " 'principaux aphicides foliaires homologues betteraves* nouveaux produits contact, inhalation, ingestion',\n",
       " \"note commune acta agpm inra atrazinel'atrazine désherbant utilisé france essentiellement cultures maïs pour l'entretien zones cultivées (voies ferrées, bordures routes, berges, etc...). suite utilisations, quelquefois observé résidus d'atrazine dans eaux supérieurs normes communautaires. donc souhaitable cette campagne promouvoir conditions d'emploi ('atrazjne visant modérer apports.\",\n",
       " \"culture mais-» eviter applications prélevée. préférer traitements postlevée association avec d'autres matières actives, permet réduire quantités d'atrazine l'hectare. dans mesure l'atrazine utilisée seule jamais dépasser dose g/ha matière active.\",\n",
       " \"zones cultivées-. réserver l'atrazine seuls traitements d'entretien. privilégier associations avec d'autres matières actives. jamais dépasser dose g/ha matière active. préférer d'autres désherbants totaux dégradation rapide. bien entendu, préconisations doivent s'appliquer sans préjudices dispositions relatives périmètres protection (immédiats, rapprochés éloignés) entourant zones captages eaux. outre, faut rappeler toutes précautions doivent être prises cours rinçage pulvérisateurs lors destruction emballages vides pour éviter entraînement vers fosses ruisseaux. enfin, certain nombre d'études sont cours réalisation sous responsabilité plusieurs ministères avec concours organisations professionnelles agricoles, sociétés distribution d'eau potable. rindustrie phytosanitaire. elles pour préciser l'origine l'évolution résidus dans l'eau, dans différentes régions selon diverses modalités d'application l'atrazine.\"]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_texts[-9:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Création de LMDataBunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:52:08 - INFO - root -   Formatting corpus for data\\lm_train.txt\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='2067' class='' max='2067' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [2067/2067 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:52:08 - INFO - root -   Formatting corpus for data\\lm_val.txt\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='230' class='' max='230' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [230/230 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:52:09 - INFO - transformers.tokenization_utils_base -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model from cache at C:\\Users\\Raja/.cache\\torch\\transformers\\3715e3a4a2de48834619b2a6f48979e13ddff5cabfb1f3409db689f9ce3bb98f.28d30f926f545047fc59da64289371eef0fbdc0764ce9ec56f808a646fcfec59\n",
      "12/10/2020 14:52:09 - INFO - root -   Loading features from cached file data\\lm_cache\\cached_camembert-base_train_256\n",
      "12/10/2020 14:52:09 - INFO - root -   Loading features from cached file data\\lm_cache\\cached_camembert-base_dev_256\n"
     ]
    }
   ],
   "source": [
    "databunch_lm = BertLMDataBunch.from_raw_corpus(\n",
    "                    data_dir=DATA_PATH,\n",
    "                    text_list=all_texts,\n",
    "                    tokenizer='camembert-base',\n",
    "                    batch_size_per_gpu=4, #was 16\n",
    "                    max_seq_length=256, #was 512\n",
    "                    multi_gpu=False,\n",
    "                    model_type='camembert-base',\n",
    "                    logger=logger)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Création de LMLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:53:34 - INFO - transformers.configuration_utils -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json from cache at C:\\Users\\Raja/.cache\\torch\\transformers\\5152a7b8b97da26abdad9b3babb600e77c52a002331ea52a9eaf96ea8b31ef8f.5bd7a9a60b9a2d311368226259eaf870cfb2248e0752f28b444ec112977cf8fc\n",
      "12/10/2020 14:53:34 - INFO - transformers.configuration_utils -   Model config CamembertConfig {\n",
      "  \"architectures\": [\n",
      "    \"CamembertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 5,\n",
      "  \"eos_token_id\": 6,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"camembert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 32005\n",
      "}\n",
      "\n",
      "12/10/2020 14:53:35 - INFO - transformers.modeling_utils -   loading weights file https://cdn.huggingface.co/camembert-base-pytorch_model.bin from cache at C:\\Users\\Raja/.cache\\torch\\transformers\\ee2c065a25e8f42cb2c52eb565b1d3757a20cf9776e93753bab41493298d70f9.b064d06bc988544d63f563021dbacad6eda41bd45190b6186beb8ebbc959013c\n",
      "12/10/2020 14:53:37 - INFO - transformers.modeling_utils -   All model checkpoint weights were used when initializing CamembertForMaskedLM.\n",
      "\n",
      "12/10/2020 14:53:37 - WARNING - transformers.modeling_utils -   Some weights of CamembertForMaskedLM were not initialized from the model checkpoint at camembert-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "lm_learner = BertLMLearner.from_pretrained_model(\n",
    "                            dataBunch=databunch_lm,\n",
    "                            pretrained_path='camembert-base',\n",
    "                            output_dir=MODEL_PATH,\n",
    "                            metrics=[],\n",
    "                            device=device_cuda,\n",
    "                            logger=logger,\n",
    "                            multi_gpu=False,\n",
    "                            logging_steps=50,\n",
    "                            is_fp16=False) #was true with gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:53:57 - INFO - root -   ***** Running training *****\n",
      "12/10/2020 14:53:57 - INFO - root -     Num examples = 961\n",
      "12/10/2020 14:53:57 - INFO - root -     Num Epochs = 10\n",
      "12/10/2020 14:53:57 - INFO - root -     Total train batch size (w. parallel, distributed & accumulation) = 4\n",
      "12/10/2020 14:53:57 - INFO - root -     Gradient Accumulation steps = 1\n",
      "12/10/2020 14:53:57 - INFO - root -     Total optimization steps = 2410\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:54:14 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:54:14 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:54:14 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:54:15 - INFO - root -   eval_loss after step 50: 0.23925103705662948: \n",
      "12/10/2020 14:54:15 - INFO - root -   eval_perplexity after step 50: 1.2702974081039429: \n",
      "12/10/2020 14:54:15 - INFO - root -   lr after step 50: 9.989383241845838e-05\n",
      "12/10/2020 14:54:15 - INFO - root -   train_loss after step 50: 2.7681958866119385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Raja\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:247: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:54:31 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:54:31 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:54:31 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:54:33 - INFO - root -   eval_loss after step 100: 0.21115561173512384: \n",
      "12/10/2020 14:54:33 - INFO - root -   eval_perplexity after step 100: 1.2351045608520508: \n",
      "12/10/2020 14:54:33 - INFO - root -   lr after step 100: 9.957578053604837e-05\n",
      "12/10/2020 14:54:33 - INFO - root -   train_loss after step 100: 2.3340562272071836\n",
      "12/10/2020 14:54:49 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:54:49 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:54:49 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:54:51 - INFO - root -   eval_loss after step 150: 0.21081974529303038: \n",
      "12/10/2020 14:54:51 - INFO - root -   eval_perplexity after step 150: 1.2346898317337036: \n",
      "12/10/2020 14:54:51 - INFO - root -   lr after step 150: 9.904719502473634e-05\n",
      "12/10/2020 14:54:51 - INFO - root -   train_loss after step 150: 2.309780428409576\n",
      "12/10/2020 14:55:07 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:55:07 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:55:07 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:55:09 - INFO - root -   eval_loss after step 200: 0.199887589766429: \n",
      "12/10/2020 14:55:09 - INFO - root -   eval_perplexity after step 200: 1.221265435218811: \n",
      "12/10/2020 14:55:09 - INFO - root -   lr after step 200: 9.831032063033726e-05\n",
      "12/10/2020 14:55:09 - INFO - root -   train_loss after step 200: 2.1669001817703246\n",
      "12/10/2020 14:55:22 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:55:22 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:55:22 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:55:24 - INFO - root -   eval_loss after epoch 1: 0.21822101565507743: \n",
      "12/10/2020 14:55:24 - INFO - root -   eval_perplexity after epoch 1: 1.2438619136810303: \n",
      "12/10/2020 14:55:24 - INFO - root -   lr after epoch 1: 9.755282581475769e-05\n",
      "12/10/2020 14:55:24 - INFO - root -   train_loss after epoch 1: 2.3389858165717223\n",
      "12/10/2020 14:55:24 - INFO - root -   \n",
      "\n",
      "12/10/2020 14:55:27 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:55:27 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:55:27 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:55:28 - INFO - root -   eval_loss after step 250: 0.20181022125941056: \n",
      "12/10/2020 14:55:28 - INFO - root -   eval_perplexity after step 250: 1.2236157655715942: \n",
      "12/10/2020 14:55:28 - INFO - root -   lr after step 250: 9.736828663974527e-05\n",
      "12/10/2020 14:55:28 - INFO - root -   train_loss after step 250: 2.0521333241462707\n",
      "12/10/2020 14:55:44 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:55:44 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:55:44 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:55:46 - INFO - root -   eval_loss after step 300: 0.19090593778170073: \n",
      "12/10/2020 14:55:46 - INFO - root -   eval_perplexity after step 300: 1.2103456258773804: \n",
      "12/10/2020 14:55:46 - INFO - root -   lr after step 300: 9.62250935917808e-05\n",
      "12/10/2020 14:55:46 - INFO - root -   train_loss after step 300: 1.9913123011589051\n",
      "12/10/2020 14:56:02 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:56:02 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:56:02 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:56:04 - INFO - root -   eval_loss after step 350: 0.17624248564243317: \n",
      "12/10/2020 14:56:04 - INFO - root -   eval_perplexity after step 350: 1.1927272081375122: \n",
      "12/10/2020 14:56:04 - INFO - root -   lr after step 350: 9.488559628808939e-05\n",
      "12/10/2020 14:56:04 - INFO - root -   train_loss after step 350: 1.961831169128418\n",
      "12/10/2020 14:56:20 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:56:20 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:56:20 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:56:22 - INFO - root -   eval_loss after step 400: 0.17745191088089576: \n",
      "12/10/2020 14:56:22 - INFO - root -   eval_perplexity after step 400: 1.194170594215393: \n",
      "12/10/2020 14:56:22 - INFO - root -   lr after step 400: 9.335548317623957e-05\n",
      "12/10/2020 14:56:22 - INFO - root -   train_loss after step 400: 1.9115200734138489\n",
      "12/10/2020 14:56:38 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:56:38 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:56:38 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:56:40 - INFO - root -   eval_loss after step 450: 0.1513343109534337: \n",
      "12/10/2020 14:56:40 - INFO - root -   eval_perplexity after step 450: 1.1633855104446411: \n",
      "12/10/2020 14:56:40 - INFO - root -   lr after step 450: 9.164125219257418e-05\n",
      "12/10/2020 14:56:40 - INFO - root -   train_loss after step 450: 1.9525598120689391\n",
      "12/10/2020 14:56:50 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:56:50 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:56:50 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:56:52 - INFO - root -   eval_loss after epoch 2: 0.1481938877931008: \n",
      "12/10/2020 14:56:52 - INFO - root -   eval_perplexity after epoch 2: 1.1597377061843872: \n",
      "12/10/2020 14:56:52 - INFO - root -   lr after epoch 2: 9.045084971874738e-05\n",
      "12/10/2020 14:56:52 - INFO - root -   train_loss after epoch 2: 1.9443070933037279\n",
      "12/10/2020 14:56:52 - INFO - root -   \n",
      "\n",
      "12/10/2020 14:56:58 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:56:58 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:56:58 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:56:59 - INFO - root -   eval_loss after step 500: 0.16651911231187674: \n",
      "12/10/2020 14:56:59 - INFO - root -   eval_perplexity after step 500: 1.1811860799789429: \n",
      "12/10/2020 14:56:59 - INFO - root -   lr after step 500: 8.975018316740278e-05\n",
      "12/10/2020 14:56:59 - INFO - root -   train_loss after step 500: 1.8591113138198851\n",
      "12/10/2020 14:57:16 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:57:16 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:57:16 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:57:17 - INFO - root -   eval_loss after step 550: 0.17550056485029367: \n",
      "12/10/2020 14:57:17 - INFO - root -   eval_perplexity after step 550: 1.1918426752090454: \n",
      "12/10/2020 14:57:17 - INFO - root -   lr after step 550: 8.769030690972262e-05\n",
      "12/10/2020 14:57:17 - INFO - root -   train_loss after step 550: 1.8182248163223267\n",
      "12/10/2020 14:57:34 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:57:34 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:57:34 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:57:35 - INFO - root -   eval_loss after step 600: 0.1876297088769766: \n",
      "12/10/2020 14:57:35 - INFO - root -   eval_perplexity after step 600: 1.206386685371399: \n",
      "12/10/2020 14:57:35 - INFO - root -   lr after step 600: 8.547037110275579e-05\n",
      "12/10/2020 14:57:35 - INFO - root -   train_loss after step 600: 1.7828229904174804\n",
      "12/10/2020 14:57:52 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:57:52 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:57:52 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:57:53 - INFO - root -   eval_loss after step 650: 0.1637064000734916: \n",
      "12/10/2020 14:57:53 - INFO - root -   eval_perplexity after step 650: 1.1778684854507446: \n",
      "12/10/2020 14:57:53 - INFO - root -   lr after step 650: 8.309980315513444e-05\n",
      "12/10/2020 14:57:53 - INFO - root -   train_loss after step 650: 1.7412176465988158\n",
      "12/10/2020 14:58:10 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:58:10 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:58:10 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:58:11 - INFO - root -   eval_loss after step 700: 0.16880887632186597: \n",
      "12/10/2020 14:58:11 - INFO - root -   eval_perplexity after step 700: 1.1838937997817993: \n",
      "12/10/2020 14:58:11 - INFO - root -   lr after step 700: 8.058867016549372e-05\n",
      "12/10/2020 14:58:11 - INFO - root -   train_loss after step 700: 1.7518587851524352\n",
      "12/10/2020 14:58:19 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:58:19 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:58:19 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:58:20 - INFO - root -   eval_loss after epoch 3: 0.16114349090136015: \n",
      "12/10/2020 14:58:20 - INFO - root -   eval_perplexity after epoch 3: 1.1748535633087158: \n",
      "12/10/2020 14:58:20 - INFO - root -   lr after epoch 3: 7.938926261462366e-05\n",
      "12/10/2020 14:58:20 - INFO - root -   train_loss after epoch 3: 1.7688201685664071\n",
      "12/10/2020 14:58:20 - INFO - root -   \n",
      "\n",
      "12/10/2020 14:58:29 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:58:29 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:58:29 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:58:31 - INFO - root -   eval_loss after step 750: 0.17748419825847334: \n",
      "12/10/2020 14:58:31 - INFO - root -   eval_perplexity after step 750: 1.1942092180252075: \n",
      "12/10/2020 14:58:31 - INFO - root -   lr after step 750: 7.794763617049124e-05\n",
      "12/10/2020 14:58:31 - INFO - root -   train_loss after step 750: 1.6648080801963807\n",
      "12/10/2020 14:58:47 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:58:47 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:58:47 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:58:49 - INFO - root -   eval_loss after step 800: 0.1689751950594095: \n",
      "12/10/2020 14:58:49 - INFO - root -   eval_perplexity after step 800: 1.1840907335281372: \n",
      "12/10/2020 14:58:49 - INFO - root -   lr after step 800: 7.518791685780768e-05\n",
      "12/10/2020 14:58:49 - INFO - root -   train_loss after step 800: 1.7063796496391297\n",
      "12/10/2020 14:59:05 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:59:05 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:59:05 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:59:07 - INFO - root -   eval_loss after step 850: 0.18472264592464155: \n",
      "12/10/2020 14:59:07 - INFO - root -   eval_perplexity after step 850: 1.2028847932815552: \n",
      "12/10/2020 14:59:07 - INFO - root -   lr after step 850: 7.232123193644957e-05\n",
      "12/10/2020 14:59:07 - INFO - root -   train_loss after step 850: 1.7173336195945739\n",
      "12/10/2020 14:59:23 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:59:23 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:59:23 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:59:25 - INFO - root -   eval_loss after step 900: 0.17355679548703706: \n",
      "12/10/2020 14:59:25 - INFO - root -   eval_perplexity after step 900: 1.189528226852417: \n",
      "12/10/2020 14:59:25 - INFO - root -   lr after step 900: 6.935975536662253e-05\n",
      "12/10/2020 14:59:25 - INFO - root -   train_loss after step 900: 1.6696666765213013\n",
      "12/10/2020 14:59:42 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:59:42 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:59:42 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:59:43 - INFO - root -   eval_loss after step 950: 0.16008633719040796: \n",
      "12/10/2020 14:59:43 - INFO - root -   eval_perplexity after step 950: 1.1736122369766235: \n",
      "12/10/2020 14:59:43 - INFO - root -   lr after step 950: 6.631606366053506e-05\n",
      "12/10/2020 14:59:43 - INFO - root -   train_loss after step 950: 1.6666693186759949\n",
      "12/10/2020 14:59:48 - INFO - root -   Running evaluation\n",
      "12/10/2020 14:59:48 - INFO - root -   Num examples = 99\n",
      "12/10/2020 14:59:48 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 14:59:49 - INFO - root -   eval_loss after epoch 4: 0.17239553653276884: \n",
      "12/10/2020 14:59:49 - INFO - root -   eval_perplexity after epoch 4: 1.1881476640701294: \n",
      "12/10/2020 14:59:49 - INFO - root -   lr after epoch 4: 6.545084971874738e-05\n",
      "12/10/2020 14:59:49 - INFO - root -   train_loss after epoch 4: 1.6890901643705565\n",
      "12/10/2020 14:59:49 - INFO - root -   \n",
      "\n",
      "12/10/2020 15:00:01 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:00:01 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:00:01 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:00:03 - INFO - root -   eval_loss after step 1000: 0.18290485785557672: \n",
      "12/10/2020 15:00:03 - INFO - root -   eval_perplexity after step 1000: 1.2007001638412476: \n",
      "12/10/2020 15:00:03 - INFO - root -   lr after step 1000: 6.320308247368286e-05\n",
      "12/10/2020 15:00:03 - INFO - root -   train_loss after step 1000: 1.5859255409240722\n",
      "12/10/2020 15:00:19 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:00:19 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:00:19 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:00:21 - INFO - root -   eval_loss after step 1050: 0.17946990407430208: \n",
      "12/10/2020 15:00:21 - INFO - root -   eval_perplexity after step 1050: 1.1965829133987427: \n",
      "12/10/2020 15:00:21 - INFO - root -   lr after step 1050: 6.003403171342563e-05\n",
      "12/10/2020 15:00:21 - INFO - root -   train_loss after step 1050: 1.611977367401123\n",
      "12/10/2020 15:00:37 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:00:37 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:00:37 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:00:39 - INFO - root -   eval_loss after step 1100: 0.17629954906610343: \n",
      "12/10/2020 15:00:39 - INFO - root -   eval_perplexity after step 1100: 1.1927952766418457: \n",
      "12/10/2020 15:00:39 - INFO - root -   lr after step 1100: 5.682236939796337e-05\n",
      "12/10/2020 15:00:39 - INFO - root -   train_loss after step 1100: 1.6183380424976348\n",
      "12/10/2020 15:00:56 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:00:56 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:00:56 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:00:57 - INFO - root -   eval_loss after step 1150: 0.1856249410372514: \n",
      "12/10/2020 15:00:57 - INFO - root -   eval_perplexity after step 1150: 1.2039706707000732: \n",
      "12/10/2020 15:00:57 - INFO - root -   lr after step 1150: 5.3581734504126494e-05\n",
      "12/10/2020 15:00:57 - INFO - root -   train_loss after step 1150: 1.5558024454116821\n",
      "12/10/2020 15:01:14 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:01:14 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:01:14 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:01:16 - INFO - root -   eval_loss after step 1200: 0.18070483666199905: \n",
      "12/10/2020 15:01:16 - INFO - root -   eval_perplexity after step 1200: 1.198061466217041: \n",
      "12/10/2020 15:01:16 - INFO - root -   lr after step 1200: 5.032588904668851e-05\n",
      "12/10/2020 15:01:16 - INFO - root -   train_loss after step 1200: 1.5782347190380097\n",
      "12/10/2020 15:01:17 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:01:17 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:01:17 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:01:19 - INFO - root -   eval_loss after epoch 5: 0.1828500651396238: \n",
      "12/10/2020 15:01:19 - INFO - root -   eval_perplexity after epoch 5: 1.2006343603134155: \n",
      "12/10/2020 15:01:19 - INFO - root -   lr after epoch 5: 5e-05\n",
      "12/10/2020 15:01:19 - INFO - root -   train_loss after epoch 5: 1.5794877852641696\n",
      "12/10/2020 15:01:19 - INFO - root -   \n",
      "\n",
      "12/10/2020 15:01:34 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:01:34 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:01:34 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:01:35 - INFO - root -   eval_loss after step 1250: 0.1820717637355511: \n",
      "12/10/2020 15:01:35 - INFO - root -   eval_perplexity after step 1250: 1.1997002363204956: \n",
      "12/10/2020 15:01:35 - INFO - root -   lr after step 1250: 4.7068659635173026e-05\n",
      "12/10/2020 15:01:35 - INFO - root -   train_loss after step 1250: 1.5057056283950805\n",
      "12/10/2020 15:01:52 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:01:52 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:01:52 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:01:53 - INFO - root -   eval_loss after step 1300: 0.17298371631365556: \n",
      "12/10/2020 15:01:53 - INFO - root -   eval_perplexity after step 1300: 1.1888467073440552: \n",
      "12/10/2020 15:01:53 - INFO - root -   lr after step 1300: 4.382387875634591e-05\n",
      "12/10/2020 15:01:53 - INFO - root -   train_loss after step 1300: 1.4909101474285125\n",
      "12/10/2020 15:02:10 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:02:10 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:02:10 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:02:11 - INFO - root -   eval_loss after step 1350: 0.17381364221756274: \n",
      "12/10/2020 15:02:11 - INFO - root -   eval_perplexity after step 1350: 1.1898337602615356: \n",
      "12/10/2020 15:02:11 - INFO - root -   lr after step 1350: 4.0605326031748645e-05\n",
      "12/10/2020 15:02:11 - INFO - root -   train_loss after step 1350: 1.4846127665042876\n",
      "12/10/2020 15:02:28 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:02:28 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:02:28 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:02:30 - INFO - root -   eval_loss after step 1400: 0.17594042305762952: \n",
      "12/10/2020 15:02:30 - INFO - root -   eval_perplexity after step 1400: 1.1923670768737793: \n",
      "12/10/2020 15:02:30 - INFO - root -   lr after step 1400: 3.742666969973463e-05\n",
      "12/10/2020 15:02:30 - INFO - root -   train_loss after step 1400: 1.422969514131546\n",
      "12/10/2020 15:02:45 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:02:45 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:02:45 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:02:46 - INFO - root -   eval_loss after epoch 6: 0.17240192339970514: \n",
      "12/10/2020 15:02:46 - INFO - root -   eval_perplexity after epoch 6: 1.1881552934646606: \n",
      "12/10/2020 15:02:46 - INFO - root -   lr after epoch 6: 3.4549150281252636e-05\n",
      "12/10/2020 15:02:46 - INFO - root -   train_loss after epoch 6: 1.4835398313415495\n",
      "12/10/2020 15:02:46 - INFO - root -   \n",
      "\n",
      "12/10/2020 15:02:48 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:02:48 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:02:48 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:02:49 - INFO - root -   eval_loss after step 1450: 0.1734319799221479: \n",
      "12/10/2020 15:02:49 - INFO - root -   eval_perplexity after step 1450: 1.1893798112869263: \n",
      "12/10/2020 15:02:49 - INFO - root -   lr after step 1450: 3.430140857051675e-05\n",
      "12/10/2020 15:02:49 - INFO - root -   train_loss after step 1450: 1.503297140598297\n",
      "12/10/2020 15:03:06 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:03:06 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:03:06 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:03:07 - INFO - root -   eval_loss after step 1500: 0.16765423806814048: \n",
      "12/10/2020 15:03:07 - INFO - root -   eval_perplexity after step 1500: 1.1825276613235474: \n",
      "12/10/2020 15:03:07 - INFO - root -   lr after step 1500: 3.124281470072597e-05\n",
      "12/10/2020 15:03:07 - INFO - root -   train_loss after step 1500: 1.4533905601501464\n",
      "12/10/2020 15:03:24 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:03:24 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:03:24 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:03:25 - INFO - root -   eval_loss after step 1550: 0.17829671502113342: \n",
      "12/10/2020 15:03:25 - INFO - root -   eval_perplexity after step 1550: 1.1951799392700195: \n",
      "12/10/2020 15:03:25 - INFO - root -   lr after step 1550: 2.8263877030925277e-05\n",
      "12/10/2020 15:03:25 - INFO - root -   train_loss after step 1550: 1.3928670382499695\n",
      "12/10/2020 15:03:42 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:03:42 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:03:42 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:03:43 - INFO - root -   eval_loss after step 1600: 0.17693207126397353: \n",
      "12/10/2020 15:03:43 - INFO - root -   eval_perplexity after step 1600: 1.1935499906539917: \n",
      "12/10/2020 15:03:43 - INFO - root -   lr after step 1600: 2.5377246225433303e-05\n",
      "12/10/2020 15:03:43 - INFO - root -   train_loss after step 1600: 1.4360487461090088\n",
      "12/10/2020 15:04:00 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:04:00 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:04:00 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:04:01 - INFO - root -   eval_loss after step 1650: 0.15615930465551522: \n",
      "12/10/2020 15:04:01 - INFO - root -   eval_perplexity after step 1650: 1.169012427330017: \n",
      "12/10/2020 15:04:01 - INFO - root -   lr after step 1650: 2.259518094870693e-05\n",
      "12/10/2020 15:04:01 - INFO - root -   train_loss after step 1650: 1.3898078250885009\n",
      "12/10/2020 15:04:13 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:04:13 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:04:13 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:04:15 - INFO - root -   eval_loss after epoch 7: 0.15136038683927977: \n",
      "12/10/2020 15:04:15 - INFO - root -   eval_perplexity after epoch 7: 1.1634159088134766: \n",
      "12/10/2020 15:04:15 - INFO - root -   lr after epoch 7: 2.061073738537635e-05\n",
      "12/10/2020 15:04:15 - INFO - root -   train_loss after epoch 7: 1.423371732482277\n",
      "12/10/2020 15:04:15 - INFO - root -   \n",
      "\n",
      "12/10/2020 15:04:19 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:04:19 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:04:19 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:04:21 - INFO - root -   eval_loss after step 1700: 0.1564323638494198: \n",
      "12/10/2020 15:04:21 - INFO - root -   eval_perplexity after step 1700: 1.169331669807434: \n",
      "12/10/2020 15:04:21 - INFO - root -   lr after step 1700: 1.9929495806431025e-05\n",
      "12/10/2020 15:04:21 - INFO - root -   train_loss after step 1700: 1.4191709518432618\n",
      "12/10/2020 15:04:38 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:04:38 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:04:38 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:04:39 - INFO - root -   eval_loss after step 1750: 0.14960667147086218: \n",
      "12/10/2020 15:04:39 - INFO - root -   eval_perplexity after step 1750: 1.1613773107528687: \n",
      "12/10/2020 15:04:39 - INFO - root -   lr after step 1750: 1.739151117239385e-05\n",
      "12/10/2020 15:04:39 - INFO - root -   train_loss after step 1750: 1.4018377029895783\n",
      "12/10/2020 15:04:56 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:04:56 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:04:56 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:04:57 - INFO - root -   eval_loss after step 1800: 0.16188797125449547: \n",
      "12/10/2020 15:04:57 - INFO - root -   eval_perplexity after step 1800: 1.1757285594940186: \n",
      "12/10/2020 15:04:57 - INFO - root -   lr after step 1800: 1.4992005114218805e-05\n",
      "12/10/2020 15:04:57 - INFO - root -   train_loss after step 1800: 1.42687264919281\n",
      "12/10/2020 15:05:14 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:05:14 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:05:14 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:05:15 - INFO - root -   eval_loss after step 1850: 0.15195928972501022: \n",
      "12/10/2020 15:05:15 - INFO - root -   eval_perplexity after step 1850: 1.1641128063201904: \n",
      "12/10/2020 15:05:15 - INFO - root -   lr after step 1850: 1.2741167622109556e-05\n",
      "12/10/2020 15:05:15 - INFO - root -   train_loss after step 1850: 1.445525529384613\n",
      "12/10/2020 15:05:32 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:05:32 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:05:32 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:05:34 - INFO - root -   eval_loss after step 1900: 0.15428919746325567: \n",
      "12/10/2020 15:05:34 - INFO - root -   eval_perplexity after step 1900: 1.1668282747268677: \n",
      "12/10/2020 15:05:34 - INFO - root -   lr after step 1900: 1.0648557334985309e-05\n",
      "12/10/2020 15:05:34 - INFO - root -   train_loss after step 1900: 1.3089814209938049\n",
      "12/10/2020 15:05:43 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:05:43 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:05:43 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:05:44 - INFO - root -   eval_loss after epoch 8: 0.15421479997726587: \n",
      "12/10/2020 15:05:44 - INFO - root -   eval_perplexity after epoch 8: 1.1667414903640747: \n",
      "12/10/2020 15:05:44 - INFO - root -   lr after epoch 8: 9.549150281252633e-06\n",
      "12/10/2020 15:05:44 - INFO - root -   train_loss after epoch 8: 1.3772374700708507\n",
      "12/10/2020 15:05:44 - INFO - root -   \n",
      "\n",
      "12/10/2020 15:05:51 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:05:51 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:05:51 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:05:53 - INFO - root -   eval_loss after step 1950: 0.14776954513329726: \n",
      "12/10/2020 15:05:53 - INFO - root -   eval_perplexity after step 1950: 1.1592457294464111: \n",
      "12/10/2020 15:05:53 - INFO - root -   lr after step 1950: 8.723060947777777e-06\n",
      "12/10/2020 15:05:53 - INFO - root -   train_loss after step 1950: 1.353246694803238\n",
      "12/10/2020 15:06:10 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:06:10 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:06:10 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:06:11 - INFO - root -   eval_loss after step 2000: 0.14849575608968735: \n",
      "12/10/2020 15:06:11 - INFO - root -   eval_perplexity after step 2000: 1.1600878238677979: \n",
      "12/10/2020 15:06:11 - INFO - root -   lr after step 2000: 6.972855472274853e-06\n",
      "12/10/2020 15:06:11 - INFO - root -   train_loss after step 2000: 1.3674042105674744\n",
      "12/10/2020 15:06:28 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:06:28 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:06:28 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:06:29 - INFO - root -   eval_loss after step 2050: 0.14699742541863367: \n",
      "12/10/2020 15:06:29 - INFO - root -   eval_perplexity after step 2050: 1.158350944519043: \n",
      "12/10/2020 15:06:29 - INFO - root -   lr after step 2050: 5.405373511777934e-06\n",
      "12/10/2020 15:06:29 - INFO - root -   train_loss after step 2050: 1.351279706954956\n",
      "12/10/2020 15:06:46 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:06:46 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:06:46 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:06:48 - INFO - root -   eval_loss after step 2100: 0.14764318214013025: \n",
      "12/10/2020 15:06:48 - INFO - root -   eval_perplexity after step 2100: 1.1590992212295532: \n",
      "12/10/2020 15:06:48 - INFO - root -   lr after step 2100: 4.027271697041252e-06\n",
      "12/10/2020 15:06:48 - INFO - root -   train_loss after step 2100: 1.3563277101516724\n",
      "12/10/2020 15:07:04 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:07:04 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:07:04 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:07:06 - INFO - root -   eval_loss after step 2150: 0.14931805661091438: \n",
      "12/10/2020 15:07:06 - INFO - root -   eval_perplexity after step 2150: 1.1610422134399414: \n",
      "12/10/2020 15:07:06 - INFO - root -   lr after step 2150: 2.844402417536374e-06\n",
      "12/10/2020 15:07:06 - INFO - root -   train_loss after step 2150: 1.345567079782486\n",
      "12/10/2020 15:07:12 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:07:12 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:07:12 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:07:13 - INFO - root -   eval_loss after epoch 9: 0.1482222005724907: \n",
      "12/10/2020 15:07:13 - INFO - root -   eval_perplexity after epoch 9: 1.1597706079483032: \n",
      "12/10/2020 15:07:13 - INFO - root -   lr after epoch 9: 2.4471741852423237e-06\n",
      "12/10/2020 15:07:13 - INFO - root -   train_loss after epoch 9: 1.3617282262481594\n",
      "12/10/2020 15:07:13 - INFO - root -   \n",
      "\n",
      "12/10/2020 15:07:24 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:07:24 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:07:24 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:07:25 - INFO - root -   eval_loss after step 2200: 0.14868027086441332: \n",
      "12/10/2020 15:07:25 - INFO - root -   eval_perplexity after step 2200: 1.160301923751831: \n",
      "12/10/2020 15:07:25 - INFO - root -   lr after step 2200: 1.861788968090683e-06\n",
      "12/10/2020 15:07:25 - INFO - root -   train_loss after step 2200: 1.327067906856537\n",
      "12/10/2020 15:07:42 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:07:42 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:07:42 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:07:43 - INFO - root -   eval_loss after step 2250: 0.14754786170445955: \n",
      "12/10/2020 15:07:43 - INFO - root -   eval_perplexity after step 2250: 1.1589887142181396: \n",
      "12/10/2020 15:07:43 - INFO - root -   lr after step 2250: 1.0836042164448945e-06\n",
      "12/10/2020 15:07:43 - INFO - root -   train_loss after step 2250: 1.311480085849762\n",
      "12/10/2020 15:08:00 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:08:00 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:08:00 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:08:01 - INFO - root -   eval_loss after step 2300: 0.1509187542475187: \n",
      "12/10/2020 15:08:01 - INFO - root -   eval_perplexity after step 2300: 1.1629021167755127: \n",
      "12/10/2020 15:08:01 - INFO - root -   lr after step 2300: 5.131528823220099e-07\n",
      "12/10/2020 15:08:01 - INFO - root -   train_loss after step 2300: 1.4004107570648194\n",
      "12/10/2020 15:08:18 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:08:18 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:08:18 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:08:20 - INFO - root -   eval_loss after step 2350: 0.14751078073795026: \n",
      "12/10/2020 15:08:20 - INFO - root -   eval_perplexity after step 2350: 1.1589457988739014: \n",
      "12/10/2020 15:08:20 - INFO - root -   lr after step 2350: 1.5285750326325954e-07\n",
      "12/10/2020 15:08:20 - INFO - root -   train_loss after step 2350: 1.3645135688781738\n",
      "12/10/2020 15:08:36 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:08:36 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:08:36 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:08:38 - INFO - root -   eval_loss after step 2400: 0.14972286614087912: \n",
      "12/10/2020 15:08:38 - INFO - root -   eval_perplexity after step 2400: 1.1615122556686401: \n",
      "12/10/2020 15:08:38 - INFO - root -   lr after step 2400: 4.248146830060362e-09\n",
      "12/10/2020 15:08:38 - INFO - root -   train_loss after step 2400: 1.3905822920799256\n",
      "12/10/2020 15:08:41 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:08:41 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:08:41 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:08:42 - INFO - root -   eval_loss after epoch 10: 0.14774326120431608: \n",
      "12/10/2020 15:08:42 - INFO - root -   eval_perplexity after epoch 10: 1.1592152118682861: \n",
      "12/10/2020 15:08:42 - INFO - root -   lr after epoch 10: 0.0\n",
      "12/10/2020 15:08:42 - INFO - root -   train_loss after epoch 10: 1.363910719319498\n",
      "12/10/2020 15:08:42 - INFO - root -   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2410, 1.6330479007538918)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_learner.fit(epochs=10, #was 30\n",
    "            lr=1e-4,\n",
    "            validate=True,\n",
    "            schedule_type=\"warmup_cosine\",\n",
    "            optimizer_type=\"adamw\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:22:29 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:22:29 - INFO - root -   Num examples = 99\n",
      "12/10/2020 15:22:29 - INFO - root -   Validation Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='13' class='' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [13/13 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'loss': 0.14940655346100146, 'perplexity': 1.1611449718475342}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_learner.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:23:16 - INFO - transformers.configuration_utils -   Configuration saved in model\\model_out\\config.json\n",
      "12/10/2020 15:23:18 - INFO - transformers.modeling_utils -   Model weights saved in model\\model_out\\pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "lm_learner.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "del lm_learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Création de databunch pour la classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:23:46 - INFO - transformers.configuration_utils -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json from cache at C:\\Users\\Raja/.cache\\torch\\transformers\\5152a7b8b97da26abdad9b3babb600e77c52a002331ea52a9eaf96ea8b31ef8f.5bd7a9a60b9a2d311368226259eaf870cfb2248e0752f28b444ec112977cf8fc\n",
      "12/10/2020 15:23:46 - INFO - transformers.configuration_utils -   Model config CamembertConfig {\n",
      "  \"architectures\": [\n",
      "    \"CamembertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 5,\n",
      "  \"eos_token_id\": 6,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"camembert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 32005\n",
      "}\n",
      "\n",
      "12/10/2020 15:23:46 - INFO - transformers.tokenization_utils_base -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model from cache at C:\\Users\\Raja/.cache\\torch\\transformers\\3715e3a4a2de48834619b2a6f48979e13ddff5cabfb1f3409db689f9ce3bb98f.28d30f926f545047fc59da64289371eef0fbdc0764ce9ec56f808a646fcfec59\n",
      "12/10/2020 15:23:46 - INFO - root -   Loading features from cached file data\\cache\\cached_camembert-base_train_multi_label_256_train_set.csv\n",
      "12/10/2020 15:23:46 - INFO - root -   Loading features from cached file data\\cache\\cached_camembert-base_dev_multi_label_256_val_set.csv\n"
     ]
    }
   ],
   "source": [
    "databunch = BertDataBunch(DATA_PATH, LABEL_PATH,\n",
    "                          tokenizer='camembert-base',\n",
    "                          train_file='train_set.csv',\n",
    "                          val_file='val_set.csv',\n",
    "                          label_file='labels.txt',\n",
    "                          text_col='report_text',\n",
    "                          label_col=['Bioagressor','Disease'],\n",
    "                          batch_size_per_gpu=8,\n",
    "                          max_seq_length=256,\n",
    "                          multi_gpu=False,\n",
    "                          multi_label=True,\n",
    "                          model_type='camembert-base')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Création de Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = [{'name': 'fbeta', 'function': fbeta}, {'name': 'roc_auc', 'function': roc_auc}]\n",
    "OUTPUT_DIR = Path('./finetuned_model')\n",
    "WGTS_PATH = Path('model/model_out/pytorch_model.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:24:20 - INFO - transformers.configuration_utils -   loading configuration file model/model_out\\config.json\n",
      "12/10/2020 15:24:20 - INFO - transformers.configuration_utils -   Model config CamembertConfig {\n",
      "  \"architectures\": [\n",
      "    \"CamembertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 5,\n",
      "  \"eos_token_id\": 6,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"camembert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 32005\n",
      "}\n",
      "\n",
      "12/10/2020 15:24:20 - INFO - transformers.modeling_utils -   loading weights file model/model_out\\pytorch_model.bin\n",
      "12/10/2020 15:24:22 - WARNING - transformers.modeling_utils -   Some weights of the model checkpoint at model/model_out were not used when initializing CamembertForMultiLabelSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.decoder.bias']\n",
      "- This IS expected if you are initializing CamembertForMultiLabelSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing CamembertForMultiLabelSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "12/10/2020 15:24:22 - WARNING - transformers.modeling_utils -   Some weights of CamembertForMultiLabelSequenceClassification were not initialized from the model checkpoint at model/model_out and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# issue fast-bert pos_weight <= downgrade to 1.9.1 solve the prob\n",
    "cl_learner = BertLearner.from_pretrained_model(\n",
    "                        databunch,\n",
    "                        pretrained_path='model/model_out',\n",
    "                        metrics=metrics,\n",
    "                        device=device_cuda, #was device_cuda\n",
    "                        logger=logger,\n",
    "                        output_dir=OUTPUT_DIR,\n",
    "                        finetuned_wgts_path=WGTS_PATH,\n",
    "                        warmup_steps=300,\n",
    "                        multi_gpu=False,\n",
    "                        multi_label=True,\n",
    "                        is_fp16=False,#True when is cuda\n",
    "                        logging_steps=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:24:46 - INFO - root -   ***** Running training *****\n",
      "12/10/2020 15:24:46 - INFO - root -     Num examples = 322\n",
      "12/10/2020 15:24:46 - INFO - root -     Num Epochs = 10\n",
      "12/10/2020 15:24:46 - INFO - root -     Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "12/10/2020 15:24:46 - INFO - root -     Gradient Accumulation steps = 1\n",
      "12/10/2020 15:24:46 - INFO - root -     Total optimization steps = 410\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='10' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/10 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='41' class='' max='41' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [41/41 00:05<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/10/2020 15:24:52 - INFO - root -   Running evaluation\n",
      "12/10/2020 15:24:52 - INFO - root -     Num examples = 80\n",
      "12/10/2020 15:24:52 - INFO - root -     Batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='5' class='' max='5' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [5/5 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "ValueError",
     "evalue": "Input contains NaN, infinity or a value too large for dtype('float32').",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-24-5e19f51fa525>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m             \u001b[0mvalidate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m             \u001b[0mschedule_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"warmup_cosine\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m             optimizer_type=\"adamw\")\n\u001b[0m",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\fast_bert\\learner_cls.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, validate, return_results, schedule_type, optimizer_type)\u001b[0m\n\u001b[0;32m    421\u001b[0m             \u001b[1;31m# Evaluate the model against validation set after every epoch\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    422\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 423\u001b[1;33m                 \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    424\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mresults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    425\u001b[0m                     self.logger.info(\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\fast_bert\\learner_cls.py\u001b[0m in \u001b[0;36mvalidate\u001b[1;34m(self, quiet, loss_only)\u001b[0m\n\u001b[0;32m    515\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mmetric\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    516\u001b[0m                 validation_scores[metric[\"name\"]] = metric[\"function\"](\n\u001b[1;32m--> 517\u001b[1;33m                     \u001b[0mall_logits\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mall_labels\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    518\u001b[0m                 )\n\u001b[0;32m    519\u001b[0m             \u001b[0mresults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalidation_scores\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\fast_bert\\metrics.py\u001b[0m in \u001b[0;36mroc_auc\u001b[1;34m(y_pred, y_true)\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m     \u001b[1;31m# Compute micro-average ROC curve and ROC area\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m     \u001b[0mfpr\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"micro\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtpr\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"micro\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mroc_curve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     77\u001b[0m     \u001b[0mroc_auc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"micro\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mauc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfpr\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"micro\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtpr\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"micro\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     70\u001b[0m                           FutureWarning)\n\u001b[0;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 72\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\sklearn\\metrics\\_ranking.py\u001b[0m in \u001b[0;36mroc_curve\u001b[1;34m(y_true, y_score, pos_label, sample_weight, drop_intermediate)\u001b[0m\n\u001b[0;32m    774\u001b[0m     \"\"\"\n\u001b[0;32m    775\u001b[0m     fps, tps, thresholds = _binary_clf_curve(\n\u001b[1;32m--> 776\u001b[1;33m         y_true, y_score, pos_label=pos_label, sample_weight=sample_weight)\n\u001b[0m\u001b[0;32m    777\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    778\u001b[0m     \u001b[1;31m# Attempt to drop thresholds corresponding to points in between and\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\sklearn\\metrics\\_ranking.py\u001b[0m in \u001b[0;36m_binary_clf_curve\u001b[1;34m(y_true, y_score, pos_label, sample_weight)\u001b[0m\n\u001b[0;32m    543\u001b[0m     \u001b[0my_score\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcolumn_or_1d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_score\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m     \u001b[0massert_all_finite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m     \u001b[0massert_all_finite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_score\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    546\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0msample_weight\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     70\u001b[0m                           FutureWarning)\n\u001b[0;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 72\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36massert_all_finite\u001b[1;34m(X, allow_nan)\u001b[0m\n\u001b[0;32m    115\u001b[0m     \u001b[0mallow_nan\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mbool\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    116\u001b[0m     \"\"\"\n\u001b[1;32m--> 117\u001b[1;33m     \u001b[0m_assert_all_finite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0msp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0missparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mallow_nan\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    118\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    119\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36m_assert_all_finite\u001b[1;34m(X, allow_nan, msg_dtype)\u001b[0m\n\u001b[0;32m     97\u001b[0m                     \u001b[0mmsg_err\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m                     (type_err,\n\u001b[1;32m---> 99\u001b[1;33m                      msg_dtype if msg_dtype is not None else X.dtype)\n\u001b[0m\u001b[0;32m    100\u001b[0m             )\n\u001b[0;32m    101\u001b[0m     \u001b[1;31m# for object dtype data, we only check for NaNs (GH-13254)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Input contains NaN, infinity or a value too large for dtype('float32')."
     ]
    }
   ],
   "source": [
    "cl_learner.fit(epochs=10,# was 30\n",
    "            lr=2e-5,\n",
    "            validate=True,\n",
    "            schedule_type=\"warmup_cosine\",\n",
    "            optimizer_type=\"adamw\")\n",
    "#WHY??????????????????????????????????"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='5' class='' max='5' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [5/5 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'loss': 0.688255774974823, 'fbeta': 0.4375, 'roc_auc': 0.42315026697177727}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cl_learner.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_learner.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "del cl_learner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prédictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor = BertClassificationPredictor(\n",
    "                model_path='finetuned_model/model_out',\n",
    "                label_path='labels/',\n",
    "                multi_label=True,\n",
    "                model_type='camembert-base',\n",
    "                do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Raja\\Documents\\SJI\\tools\\anaconda3\\envs\\fast-camembert\\lib\\site-packages\\pandas\\core\\frame.py:1490: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
      "  FutureWarning,\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('disease', 0.336669921875), ('bioagressor', 0.277587890625)]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 1 - cicadelle\n",
    "predictor.predict(\"Cicadelles La cicadelle Edwardsiana est toujours observée sur les parcelles en été\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.33740234375), ('bioagressor', 0.280029296875)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.predict(\"election américane\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.33935546875), ('bioagressor', 0.283935546875)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 1, bioagressor: 0 - mildiou\n",
    "predictor.predict(\"vigilant : en particulier vis-à-vis du mildiou, de l’oïdium et de la bactériose / cladosporiose. Les prévisions pour les prochains jours restent peu favorables à l’expression de la bactériose et la cladosporiose, si elles se confirment. Mildiou (Pseudoperonospora cubensis) : Le modèle annonce un risque élevé pour toutes les dates de plantation avec les données de la station de Thurageau. Avec les données de la station de Maulay, les plantations en S25 et S26 montrent un risque modéré. Le risque est un peu plus élevé dans le sud de la Charente- Maritime que dans le Poitou. Niveau de risque Faible Moyen Élevé Très élevé Indice : Log (Nb de taches/unité de surface) -14 à -9 -9 à -4 -4 +4 Équivalent en unité de surface 1 tâche par hectare par 100 m2 1 tâche par 100 m2 par m2 1 tâche par m2à 1 % de surface atteinte 1 % à 100 % de surface atteinte Évaluation du risque : les conditions restent favorables à ce microorganisme (qui n’est pas un champignon, mais proche d’une algue). Les BSV sont disponibles en accès direct sur le site  (rubrique : Nos publications - Bulletin de santé du végétal) ou par abonnement en ligne gratuit sur le site  BSV CULTURES LÉGUMIÈRES DE PLEIN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.3359375), ('bioagressor', 0.27490234375)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - texte sur trump trump\n",
    "predictor.predict(\"L'avance de Donald Trump dans cet Etat où 4,9 millions d'électeurs ont voté, a fondu vendredi 6 novembre. Le candidat républicain compte\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.33544921875), ('bioagressor', 0.280029296875)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - texte sur mouche de carotte mais pas de rique\n",
    "predictor.predict(\"mouche de la carotte :ajouter trichloronate\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - oidium : pas d'intervention => pas bon\n",
    "predictor.predict(\"oïdium : pas d'intervention dans l'immediat les conditions très chaudes et l'absence de rosées nocturnes sont défavorables à cette maladie. aucun symptôme n'a encore été observé.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 1, bioagressor: 0 - oidium  => good\n",
    "predictor.predict(\"oïdium du pommierce champignon est à l'heure actuelle en pleine fructification. il est nécessaire d'ajouter un antioïdium aux bouillies pour assurer'la protection du jeune feuillage.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - mildiou : pas d'intervention :=> pas bon\n",
    "predictor.predict(\"mildiou : pas d'intervention dans l'immediat les conditions très chaudes et l'absence de rosées nocturnes sont défavorables à cette maladie. aucun symptôme n'a encore été observé.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - pucerons- le seuil de nuisibilité  n’est pas atteint => raté\n",
    "predictor.predict(\"pucerons les pucerons verts sont toujours présents dans certaines parcelles. on note également la présence de sitobion avenae, relativement moins fréquente, mais avec des colonies de plusieurs dizaines d'individus. le temps frais actuel n'est pas favorable aux prédateurs, mais on observe un nombre important de pucerons «momifiés» par la ponte de parasitoïdes. le seuil de nuisibilité n'est atteint dans aucune parcelle du réseau\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - zero mot clé\n",
    "predictor.predict(\"par jour. Sur les autres secteurs, en production de carotte, les captures sont nulles. Évaluation du risque\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 + mildiou\n",
    "predictor.predict(\"7 Action pilotée par le ministère chargé de l’agriculture mildiou, avec l’appui financier de l’Office National de l’Eau et des Milieux Aquatiques (ONEMA), par les crédits issus de la redevance pour pollutions diffuses attribués au\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0\n",
    "predictor.predict(\"financement du plan Ecophyto. Ce bulletin est rédigé par l'ACPEL avec la collaboration de référents par culture (techniciens des Chambres d'Agriculture de la Charente, de la Charente-Maritime et d'Indre et Loire et de la Vienne) sur la base d'observations réalisées par des producteurs et techniciens : Charentes- Alliance, les entreprises de production de melon, la coopérative AGROLEG, la coopérative UNIRE, des producteurs d'Agrobio Poitou-Charentes. Ce bulletin est réalisé à partir d'observations ponctuelles. Il a pour vocation de donner une tendance de la situation sanitaire régionale. Celle-ci ne peut être transposée telle quelle dans les parcelles de production légumières (conditions très variables). La Chambre Régionale d'Agriculture de Poitou-Charentes et le rédacteur dégagent toute responsabilité quant aux décisions prises par les producteurs pour la protection de leurs cultures. Elle les invite à prendre ces décisions sur la base des observation s qu'ils auront réalisées dans leurs parcelles. Les\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - french crop usasge\n",
    "predictor.predict(\"Ensemble de plantes cultivées pour leurs fruits ou leurs graines riches en matières grasses (lipides). De ces fruits et graines sont extrait une huile à usage alimentaire humaine, alimentaire animal ou industriel. Les résidus de l'extraction constituent les tourteaux utilisés pour l'alimentation animale.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 0 - french crop usasge\n",
    "predictor.predict(\"Orge semé après le 1er février, principalement de mars à mai. Orge de printemps est toujours à deux rangs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 1, bioagressor: 0 - tweet - jaunisse\n",
    "predictor.predict(\"des parcelles qui ont eu une croissance presque nulle depuis cet été en majeur partie dû à la jaunisse.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 1, bioagressor: 0 - tweet - sécheresse\n",
    "predictor.predict(\"Parfois,on nous demande. Que faites-vous,par cette sécheresse?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cas disease: 0, bioagressor: 1 - tweet - pucerons\n",
    "\n",
    "predictor.predict(\"Attention pucerons dans les blés\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('disease', 0.1798095703125), ('bioagressor', 0.0885009765625)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.predict(\"Favorisée par les conditions climatiques,l'activité du Carpocapse est toujours intense et il est conseillé de renouveler la protection des fruits en effectuant un nouveau traitement dès réception de ce bulletin.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fast_cmb_kernel",
   "language": "python",
   "name": "fast_cmb_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}