notebooks/03_Create_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import TextLMDataBunch as lmdb, load_data\n",
    "from fastai.text.learner import language_model_learner\n",
    "from fastai.text.models import AWD_LSTM\n",
    "from fastai.callbacks import EarlyStoppingCallback, SaveModelCallback, ReduceLROnPlateauCallback, CSVLogger\n",
    "from fastai.train import ShowGraph\n",
    "import pandas as pd\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/ds/notebooks/../lang_model')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = Path('../lang_model/')\n",
    "path.absolute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load DataBunch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You have to redefine or import any custom you defined in the last step, because the data bunch is going to look for that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pass_through(x):\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = load_data(path, bs=120)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Instantiate Language Model\n",
    "\n",
    "We are going to use the `awd_lstm` with the default parameters listed in the config file below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data=data_lm,\n",
    "                               arch=AWD_LSTM,\n",
    "                               pretrained=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Train Language Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the best learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl81dWd//HXJwuBsIQlAcO+CO6IGlGkKlq1VKrWVp2hy7h1qLbWOlo7XWbazvizy9jWqbWt0hGpjrVTl7ZqbdXaKi4gggKiqCiLhiULgSxkTz6/P+43ENNskNzv9y7v5+NxH9x77vne88mXJJ+c7znfc8zdERGR9JURdQAiIhItJQIRkTSnRCAikuaUCERE0pwSgYhImlMiEBFJc0oEIiJpTolARCTNKRGIiKS5rKgD6I38/HyfPHly1GGIiCSV1atXl7t7QU/1kiIRTJ48mVWrVkUdhohIUjGzrb2pF7dLQ2a2xMxKzWx9h/IvmdmbZva6mf1XvNoXEZHeiecYwVJgfvsCMzsDuAA41t2PAn4Yx/ZFRKQX4pYI3H0ZUNGh+Grg++7eENQpjVf7IiLSO2HPGpoBnGpmL5nZs2Z2Ysjti4hIB2EPFmcBI4GTgROB35rZVO9kUwQzWwQsApg4cWKoQYqIpJOwewTFwMMesxJoBfI7q+jui929yN2LCgp6nP0kIiIHKexE8HvgDAAzmwEMAMpDjkFERNqJ5/TR+4HlwGFmVmxmVwJLgKnBlNLfAJd2dllIRCTdlVTV86Mn32JTWU3c24rbGIG7L+zirc/Eq00RkVSxdVctP/3rO5w0ZRRTC4bEtS2tNSQikoD21DYCkDcoO+5tKRGIiCSgyromQIlARCRt7UsEuUoEIiJpqbKuCTMYmhP/272UCEREElBlXRN5g7LJyLC4t6VEICKSgNoSQRiUCEREEtCeWiUCEZG0ph6BiEiaq1IiEBFJb3uUCERE0pe7U1nXxPAQ7iEAJQIRkYRT09BMS6urRyAikq7CXF4ClAhERBLO/kQwIJT2lAhERBJMZa16BCIiaa2tR6DBYhGRNKUxAhGRNLcnVRKBmS0xs9Jgf+K2su+Y2TYzWxM8zo1X+yIiyaqyronsTCN3QGYo7cWzR7AUmN9J+a3uPit4PB7H9kVEklLbgnNm8V+CGuKYCNx9GVARr88XEUlVVXVNDAvpshBEM0ZwjZmtCy4djYigfRGRhFZZ18TwFE4EvwCmAbOAHcCPuqpoZovMbJWZrSorKwsrPhGRyO2pawxtoBhCTgTuXuLuLe7eCvwSmN1N3cXuXuTuRQUFBeEFKSISsTD3IoCQE4GZFbZ7eSGwvqu6IiLpqrK2ieG54SwvAZAVrw82s/uBeUC+mRUD3wbmmdkswIEtwOfj1b6ISDJqaXWq6ptDHSyOWyJw94WdFN8Vr/ZERFJBdX2wvESqXhoSEZHuhb28BCgRiIgklD0hrzwKSgQiIgkl7JVHQYlARCShhL3gHCgRiIgkFI0RiIikuaogEaT6WkMiItKFPbWNDMzOYGB2OEtQgxKBiEhCCXt5CVAiEBFJKLGVR8NbXgKUCEREEkrbpjRhUiIQEUkglXVN5IV4DwEoEYiIJBSNEYiIpDklAhGRNNbY3EptY0uoK4+CEoGISMLYd1exxghERNJTFMtLgBKBiEjCUCIQEUlzlXWNgBKBiEja2r8XQYrcWWxmS8ys1MzWd/LeDWbmZpYfr/ZFRJJNZQS7k0F8ewRLgfkdC81sAnAO8F4c2xYRSTptm9IMG5gVartxSwTuvgyo6OStW4GvAh6vtkVEklFlXRNDc7LIygz3qn2orZnZBcA2d18bZrsiIsmgsrYp1A1p2oTW/zCzXOAbxC4L9ab+ImARwMSJE+MYmYhIYohieQkIt0cwDZgCrDWzLcB44BUzO6Szyu6+2N2L3L2ooKAgxDBFRKJRWdfE8JDvKoYQewTu/howuu11kAyK3L08rBhERBLZnrompo8eEnq78Zw+ej+wHDjMzIrN7Mp4tSUikgqiujQUtx6Buy/s4f3J8WpbRCTZuHskm9KA7iwWEUkIdU0tNDa3hr5fMSgRiIgkhPLq2DpD+UOUCERE0lJZTQMA+UNzQm9biUBEJAGUB4mgYIgSgYhIWmpLBPlKBCIi6altjGCUxghERNJTeU0Dw3OzyQ55wTlQIhARSQhl1Q2RXBYCJQIRkYRQXtMQydRRUCIQEUkIsUSgHoGISNoqr2lUIhARSVf1TS3UNDRTEMHNZKBEICISubLqtnsINEYgIpKWoryZDJQIREQiV17TtuCcEoGISFoqj3DBOVAiEBGJXHkwRjBqsMYIRETSUnlNA0MHZjEwOzOS9uO5Z/ESMys1s/Xtym4ys3VmtsbMnjSzsfFqX0QkWZTXNEay/HSbePYIlgLzO5Td4u4z3X0W8BjwrTi2LyKSFMoivKsY4pgI3H0ZUNGhrKrdy8GAx6t9EZFkUV7TQP7QaMYHALLCbtDMbgb+CagEzgi7fRGRRFNe3UD+ofmRtR/6YLG7f9PdJwD3Add0Vc/MFpnZKjNbVVZWFl6AIiIhamhuoaq+OWXHCHpyH/DJrt5098XuXuTuRQUFBSGGJSISnl1tN5NFdA8BhJwIzGx6u5cXAG+G2b6ISKKJenkJiOMYgZndD8wD8s2sGPg2cK6ZHQa0AluBq+LVvohIMtifCFJwsNjdF3ZSfFe82hMRSUZtm9an5PRRERHpWVnQI4hqLwJQIhARiVR5TQNDcqJbXgKUCEREIhXbojK68QFQIhARiVR5dbTLS4ASgYhIpMojXmcIlAhERCIV9TpDoEQgIhKZppZWdtc2JUePwMymmVlO8HyemV1rZsPjG5qISGqr2Bv9PQTQ+x7BQ0CLmR0KLAYmAL+OW1QiImmgrDr65SWg94mg1d2bgQuBn7r7jUBh/MISEUl95ftuJkuOMYImM1sIXEpsZzGA7PiEJCKSHsprkuvS0OXAHOBmd99sZlOAe+MXlohI6kuUS0O9WnTO3d8ArgUwsxHAUHf/QTwDExFJdeU1DQzKzmRwTuibRX5Ab2cNPWNmw8xsJPAK8Esz+3F8QxMRSW2JcA8B9P7SUF6w8fwngHvc/STgrPiFJSKS+hLhrmLofSLIMrNC4BL2DxaLiEgflFc3JlUi+E/gCeBdd3/ZzKYCG+MXlohI6iuprmfMsOgTQW8Hix8AHmj3ehPdbDwvIiLdq29qYU9tE4cMGxh1KL0eLB5vZr8zs9Lg8ZCZjY93cCIiqaq0KjZ1dEyyJALgbuARYGzweDQo65KZLQmSxvp2ZbeY2Ztmti5ILFqvSETS0s6qeiC5EkGBu9/t7s3BYylQ0MMxS4H5HcqeAo5295nA28DXDyRYEZFU0ZYIDslLnkSwy8w+Y2aZweMzwK7uDnD3ZUBFh7IngzWLAFYAurwkImmpNAl7BFcQmzq6E9gBXARc1se2rwD+1MfPEBFJSjsr6xmYncGwgdHeVQy9TATuvtXdz3f3Ancf7e4fpw+zhszsm0AzcF83dRaZ2SozW1VWVnawTYmIJKSdVfUcMmwgZhZ1KH3aoez6gznIzC4DPgZ82t29q3ruvtjdi9y9qKCgp+EIEZHkUlrVkBCXhaBvieCA05iZzQe+Cpzv7rV9aFtEJKntrKpPiUTQ5V/zAGZ2P7AcOMzMis3sSuB2YCjwlJmtMbM7+tC+iEhScndKquoTYsYQ9HBnsZlV0/kvfAMGdXesuy/spPiu3ocmIpKaKuuaaGhuTZgeQbeJwN2HhhWIiEi62H8zWfTrDEHfLg2JiMhBKAmWl0iEdYZAiUBEJHQllYlzMxkoEYiIhK7t0tBoXRoSEUlPJVX1jBw8gJyszKhDAZQIRERCV1JVz+ihidEbACUCEZHQ7UygewhAiUBEJHQlVQ0JM2MIlAhERELV1NJKeU0Do5UIRETSU1l1A+6Jcw8BKBGIiISqZN/OZBosFhFJS22JYPRQ9QhERNLSzsrE2au4jRKBiEiISqobyM40RuYOiDqUfZQIRERCVFJZz+ihA8nIiH6LyjZKBCIiIYrtTJY4A8WgRCAiEqqSBNqiso0SgYhIiEoSaNP6NnFLBGa2xMxKzWx9u7KLzex1M2s1s6J4tS0ikohqGpqpaWhOqBlDEN8ewVJgfoey9cAngGVxbFdEJCGVJNgWlW263bO4L9x9mZlN7lC2AcAscUbLRUTCkmg7k7WJWyJIBN/70wYeXFUcWfsHnu+6PqDjZ/X00e3rW1C7razjsR0Ts1nwaHecBfX21exQ1lY/duz+sgyzDxyfEZS1lWeYkZHRVs/INMjMiL2flWlkZmQEZRlkZxrZmRlkZRoDMjPIyc4kJyuDnKwMBudkkTcoe99jcv5ghuSk9Le3JKG2nckSaZ0hSOBEYGaLgEUAEydOPKjPOGpsHnuPad732v1gfjkfHPcDrH9An9X9h7ev3/bcg2M6fpZ3Vs/bl8eO9E7KYvU89l67563Bm23HtHpbeewTWt1paY2VtbQ6TS1OizutDq2tsffa6rS0Os3Bv00trTS3Ok3NrTS0tNLY3NrlOTCDyaMGc2ThMI4cO4y5h+Yzc1xeQs3dlvTTtmm9egS95O6LgcUARUVFB/hrNeb8Y8dy/rFj+zUuSRzuTmNLK/VNrextaKayronKuiZ2723kndIa3thRxfrtlfzxtR3c8sRbjBo8gNNnFHDG4aM58/DRDFaPQUJWUlXP0JyshPveS6xoRA6AmZGTlUlOViZ5g7IZO3xQp/Uq9jby3MYy/vZmKX97q5SHX91GTlYGZx4+mgUzCznz8NHkDtCPgsTfexW1jBvR+fdplOL23W9m9wPzgHwzKwa+DVQAPwUKgD+a2Rp3/0i8YhABGDl4ABfMGscFs8bR0uqs3rqbP67bzuPrd/Kn9TvJHZDJ/KMO4cLjx3HKtHwydflI4mRjaTXHjh8edRh/J56zhhZ28dbv4tWmSE8yM4zZU0Yye8pIvnXeUazcXMEja7fx2LodPPzqNsYMy+Hjx43jkqIJTCsYEnW4kkJqG5t5v6KOi0+YEHUof0f9YUlbmRnGnGmjmDNtFN8+7yie3lDKw68U8z/PbebOZzdx4uQRXFI0gQUzC3XpSPrs3dK9AEwfnXh/YOi7WwQYmJ3JgpmFLJhZSGlVPQ+9so0HVr3PjQ+u46bH3uCSogl8ds4kJo0aHHWokqTeLqkGYPqYoRFH8veUCEQ6GD1sIFfPm8ZVp09l5eYK7l2xlaUvbuGuFzZzxmGjuer0acyeMjLqMCXJbCytITvTmDQqN+pQ/o4SgUgXzIyTpo7ipKmj2FlZz69Xvsd9K7ZyyZ3LmT1lJNeeOZ25h47SnfLSKxtLqpmaP4TszMRb6zPxIhJJQIfkDeT6s2fw/L+eybfPO5L3dtXymbte4hO/eJEVm3ZFHZ4kgbdLq5k+JvHGB0CJQOSADBqQyeVzp/DsV+dx84VHs7Oynn9cvILP/epl3imtjjo8SVC1jc0U765j+ujEGx8AJQKRg5KTlcmnT5rE374yjxs/chgrNlVwzq3L+Lffv0ZlbVPU4UmCebd0L+4wQz0CkdQzMDuTL55xKM/eOI9/mjOZX7/0Hh/+8bM8tm47fqALTknK2ljaNmNIiUAkZY0aksN3zj+KR675EIV5A7nm169yxdKXKd5dG3VokgDeLmmbMZSY04+VCET60dHj8vjdF07h3xYcwUubK5j/38/x4Opi9Q7S3DuliTtjCJQIRPpdVmYGnzt1Kk9cdxpHFg7jKw+s5er/fYWKvY1RhyYRebukhkMT9LIQKBGIxM2Ekbncv+hkvv7Rw/nrm6Wcc+synn27LOqwJGR1jS28v7uWGQk6YwiUCETiKjPD+Pzp0/jDNXMZNXgAly5Zyff+tIGmlq431ZHU8m5ZDe6JO1AMSgQioTiicBh/uGYunzppInc+u4lL7lzO+xUaSE4HbWsMJerUUVAiEAnNwOxMvnvhMfzsU8fzTkkNC257jj+v3xl1WBJn+9cYSswZQ6BEIBK6BTML+eO1pzI5fzBX/e9q/vPRN7rdf1mS28aSaqbkD07YGUOgRCASiYmjcnngqjlcdspklrywmYt1qShlbSytScilp9tTIhCJSE5WJt85/yh+8enj2VQau1T05Ou6VJRK6hpbeK+iNiE3o2lPiUAkYh89ppDHrv0Qk0YNZtG9q/mPR1/XpaIU0TZjaEa69gjMbImZlZrZ+nZlI83sKTPbGPw7Il7tiySTSaMG8+DVsUtFd7+whYvveFGXilJA2xpDiTxjCOLbI1gKzO9Q9jXgaXefDjwdvBYR9l8quuMzx7OpfC/n/uQ5/rBmW9RhSR+seW8PuQMyE3rGEMQxEbj7MqCiQ/EFwK+C578CPh6v9kWS1fyjC3n82lOZPmYIX/7NGq7/7RpqGpqjDksOwopNFZwwaURCzxiC8McIxrj7juD5TmBMyO2LJIUJI3P57efncO2Hp/P7V7ex4LbnePW93VGHJQegYm8jb5VUc/LUUVGH0qPI0pTHlmPscklGM1tkZqvMbFVZmdZnkfSTlZnB9WfP4P8+P4fmFueiO5Zz61Nva3mKJLFyc2wL05Onjow4kp6FnQhKzKwQIPi3tKuK7r7Y3YvcvaigoCC0AEUSzYmTR/L4l0/l/GPH8pOnN3LRHcvZVFYTdVjSgxWbKhiUnckx44ZHHUqPwk4EjwCXBs8vBf4QcvsiSSlvUDa3/sMsbv/UcWwp38uC257n3uVbaG3VPgeJasWmXZwwaQQDshJ7fADiO330fmA5cJiZFZvZlcD3gbPNbCNwVvBaRHrpYzPH8sR1p1E0eQT//ofX+eySl7QLWgLavbeRN3dWJ8VlIYCseH2wuy/s4q0Px6tNkXRwSN5A7rliNvevfJ+b//gG8//7Of5twRH8w4kTMLOowxNg5ZbYhMmTkmCgGHRnsUhSMjM+ddJE/nzdaRwzLo+vPfwal939Mjsq66IOTYhdFhqYncHM8XlRh9IrSgQiSWzCyFzu+9xJ/Mf5R7FycwXn/HgZv335fe2RHLGXNlVw/MQR5GRlRh1KrygRiCS5jAzj0lMm8+frTuWIscP46kPruHzpy2zbo95BFCprm9iwsyop7h9oo0QgkiImjRrMb/75ZL5z3pGs3FzB2T9+lrue30yLZhaFauWWCtzhpCnJMVAMSgQiKSUjw7hs7hSe/JfTOGnKSG567A0u/PkLrN9WGXVoaWPFpl3kZGVw7ITEv3+gjRKBSAoaPyKXJZedyO2fOo7te+o5//bnuemxN7RmUQhe2ryL4yYOZ2B2cowPgBKBSMoyMz42cyxPX386C2dPZMkLmznrR8/y+Gs7NJgcJ5V1Tby+PbnGB0CJQCTl5eVmc/OFx/Dw1acwcvAAvnDfK1x698u8U6plKvrbcxvLcIc5SgQikoiOmziCR66Zy79/7Ehe3bqb+f+9jJsee4PKuqaoQ0sZj6zZzuihORRNTp6BYlAiEEkrWZkZXPmhKfztxnlcdMJ4lrywmTN/+Az3r3xPs4v6qLKuiWfeKmPBzEIyM5LrDm8lApE0lD8kh+9/ciaPXvMhpuQP5usPv8b5tz/Pys0d95KS3npi/U4aW1q5YNa4qEM5YEoEImns6HF5PHDVHH668Dh2723kkjuXc82vX9FCdgfhkbXbmTQql2OTZFmJ9pQIRNKcmXHesWN5+oZ5XHfWdP6yoYQP/+hZbnniTU037aXS6npefLec82aOTcqF/5QIRASAQQMyue6sGfz1hnmce0whP/vbu8y75Rl+s/I9mrUrWrceX7eDVocLZo2NOpSDokQgIh8wdvggbv2HWfz+i3OZNCqXrz38GvN/8hxPvr5T9x904Q9rt3P4IUOZPmZo1KEcFCUCEenUrAnDefCqOdzxmeNpdWfRvau56I7lGlDu4P2KWl59bw/nJ2lvAJQIRKQbZsb8owt58rrT+N4njqF4dy2X3LmcS5esZF3xnqjDSwiPrN0OwHkzlQhEJIVlZWawcPZEnvnKGXzj3MNZV7yH829/gX++ZxUbdlRFHV5k3J1H1mznhEkjmDAyN+pwDlokicDMvmxm683sdTO7LooYROTADRqQyaLTpvHcv57JDWfPYMWmXXz0J8/xuV+tYu376ddDeGTtdt4qqeaTx4+POpQ+sbAHf8zsaOA3wGygEfgzcJW7v9PVMUVFRb5q1aqQIhSR3qqsbWLpi1tY8sJmKuuaOHV6PlfPm8acqaOSchrlgSitqufsW5cxtWAwD151SkLeTWxmq929qKd6UfQIjgBecvdad28GngU+EUEcItJHebnZfPms6bzwtTP52kcPZ8OOKj71y5dYcNvzPLS6mMbm1Jx26u5843evUd/Uwg8vPjYhk8CBiCIRrAdONbNRZpYLnAtMiCAOEeknQ3KyuOr0aTz/r2fy/U8cQ1NLKzc8sJYP/eCv3Pb0RsqqG6IOsV89/Mo2/rKhlBs/chjTCoZEHU6fhX5pCMDMrgS+AOwFXgca3P26DnUWAYsAJk6ceMLWrVtDj1NEDo67s2xjOXc9v5llb5cxIDODBTMLueyUyUm1c1dndlbWc86tzzJjzFD+7/NzEro30NtLQ5Ekgg8EYPZdoNjdf95VHY0RiCSvd8tquOfFLTy4upi9jS0cOz6Pz86ZzMdmFibVLl4ALa3O5UtfZuXmXfzpy6cxJX9w1CF1K6ETgZmNdvdSM5sIPAmc7O5dTjlQIhBJftX1TTy0uph7V2zl3bK9jMjN5pKiCVxcNJ5DRyfHHbnfe3wDdy7bxM0XHs2nT5oUdTg9SvRE8BwwCmgCrnf3p7urr0QgkjrcnRff3cU9y7fwlw2ltLQ6syYM56ITxnPezLHk5WZHHWKnHlxdzFceWMtnT57ETR8/OupweiWhE8GBUiIQSU2l1fU8smY7D6wq5q2SarIzjbmH5nPuMYWcc+QYhucOiDpEAFZvrWDh4pc4ccoIll4+m+zM5LgXV4lARJKGu/P69ioeXbudx9btYNueOrIyjDnTRvGRow7hnKPGMHrowEhiK95dy8d/9gJDcrL4/RfnJkxy6g0lAhFJSu7OuuJKHn9tB0+8vpMtu2oxg+MnjmDejALmTs9n5rg8suL8V3l9Uwv3LN/C7X99Bwd+94W5HDo6uaaKKhGISNJzd94qqeaJ9SU8tWEn67fF1jUampPF7CkjOWZ8HkcWDuPIscMYN3xQv9zN3NzSymPrdnDLE2+xbU8d8w4r4JvnHpGUS0wrEYhIyqnY28jyd3fx/DvlvLR5F5vL99L2K2zYwCwOLxzG4YcM5fBDhjGtYDBjhw/ikLyB3V7Tb2xuZWdlPS+8W86yt8t44Z1yquqbOXrcML7+0SOYe2h+SF9d/1MiEJGUV9vYzJs7q3ljexVv7KjirZ3VvLWz+gNbbJpBwZAchuRkgUFbn2FvQwuVdU3UNbXsq1uYN5DTphdw5hGjOfuIMWQk8M1ivdHbRJAVRjAiIvGQOyCL4yeO4PiJI/aVuTvFu+vYXL6XHZV1bN9Tz47KOmobW3CA4G/fwTmZ5A3KJm9QNiMH5zB7ygimFQxJ+cXyOqNEICIpxcyYMDI3qfcHCFtyTIYVEZG4USIQEUlzSgQiImlOiUBEJM0pEYiIpDklAhGRNKdEICKS5pQIRETSXFIsMWFmlcDGTt7KAyp7+bqn5/lA+UGE17HN3r7fWfmBxt/X2LuLr6f3de4T59x3F2/712Ge++7q9PV7R+e+9+d+krsX9PiJ7p7wD2Bxb8q7e93Tc2BVf8Z2oLEfTPx9jb0/49e5T5z4u/pawjz38fze0bk/+HPf1SNZLg092svy7l735vnB6On43sbeWVlPMfc19t58hs594p/7jmVdfS1hnvvu6vT1e0fnvn/q7JMUl4bCYGarvBer9CWiZI4dkjv+ZI4dkjv+ZI4dEiv+ZOkRhGFx1AH0QTLHDskdfzLHDskdfzLHDgkUv3oEIiJpTj0CEZE0l3KJwMyWmFmpma0/iGNPMLPXzOwdM7vN2u1QYWZfMrM3zex1M/uv/o36AzH0e/xm9h0z22Zma4LHuf0f+b4Y4nL+g/dvMDM3s7jsHRinc3+Tma0LzvuTZja2/yPfF0M84r8l+L5fZ2a/M7Ph/R953GK/OPh5bTWzfr8W35eYu/i8S81sY/C4tF15tz8X/eJgp18l6gM4DTgeWH8Qx64ETia2m92fgI8G5WcAfwFygtejkyz+7wBfSdbzH7w3AXgC2ArkJ0vswLB2da4F7kimcw+cA2QFz38A/CCJYj8COAx4BihKlJiDeCZ3KBsJbAr+HRE8H9Hd19efj5TrEbj7MqCifZmZTTOzP5vZajN7zswO73icmRUS+6Fd4bGzfw/w8eDtq4Hvu3tD0EZpksUfmjjGfyvwVfZtNJgcsbt7Vbuqg5Mw/ifdvW0D4BXA+CSKfYO7vxWPePsScxc+Ajzl7hXuvht4Cpgf1s91yiWCLiwGvuTuJwBfAX7eSZ1xQHG718VBGcAM4FQze8nMnjWzE+Ma7d/ra/wA1wTd+yVmNoJw9Sl+M7sA2Obua+MdaCf6fO7N7GYzex/4NPCtOMbamf743mlzBbG/SMPSn7GHpTcxd2Yc8H67121fRyhfX8rvWWxmQ4BTgAfaXVrLOcCPySLWZTsZOBH4rZlNDTJ0XPVT/L8AbiL21+hNwI+I/VDHXV/jN7Nc4BvELlGEqp/OPe7+TeCbZvZ14Brg2/0WZDf6K/7gs74JNAP39U90PbbXb7GHpbuYzexy4MtB2aHA42bWCGx29wvDjrWjlE8ExHo9e9x9VvtCM8sEVgcvHyH2y7J9t3c8sC14Xgw8HPziX2lmrcTWCSmLZ+CBPsfv7iXtjvsl8Fg8A+6gr/FPA6YAa4MfrvHAK2Y22913JnjsHd0HPE5IiYB+it/MLgM+Bnw4jD9+Av197sPQacwA7n43cDeAmT0DXObuW9pV2QbMa/d6PLGxhG2E8fX196BDIjyAybQbwAFeBC4OnhtwbBfHdRyUOTcovwr4z+D5DGJdOEui+Avb1fkX4DfJdP471NlCnAaL43Tup7er8yXgwWQ698B84A2gIJ5xx/P7hjgNFh9szHQ9WLyZ2EDxiOANggSPAAADr0lEQVT5yN58ff3ydcT7PzfsB3A/sANoIvaX/JXE/qL8M7A2+Kb+VhfHFgHrgXeB29l/w90A4H+D914Bzkyy+O8FXgPWEfsrqjCZ4u9QZwvxmzUUj3P/UFC+jtj6L+OS6dwD7xD7w2dN8IjLrKc4xX5h8FkNQAnwRCLETCeJICi/Ijjf7wCXH8jPRV8furNYRCTNpcusIRER6YISgYhImlMiEBFJc0oEIiJpTolARCTNKRFIUjKzmpDb+x8zO7KfPqvFYquRrjezR3ta0dPMhpvZF/qjbZHOaPqoJCUzq3H3If34eVm+f3G1uGofu5n9Cnjb3W/upv5k4DF3PzqM+CT9qEcgKcPMCszsITN7OXjMDcpnm9lyM3vVzF40s8OC8svM7BEz+yvwtJnNM7NnzOxBi63Bf1/b2u9BeVHwvCZYSG6tma0wszFB+bTg9Wtm9v962WtZzv7F9YaY2dNm9krwGRcEdb4PTAt6EbcEdW8MvsZ1ZvYf/XgaJQ0pEUgq+Qlwq7ufCHwS+J+g/E3gVHc/jtjqn99td8zxwEXufnrw+jjgOuBIYCowt5N2BgMr3P1YYBnwz+3a/4m7H8MHV4zsVLBuzoeJ3e0NUA9c6O7HE9sD40dBIvoa8K67z3L3G83sHGA6MBuYBZxgZqf11J5IV9Jh0TlJH2cBR7Zb+XFYsCJkHvArM5tObAXW7HbHPOXu7deUX+nuxQBmtobYWjLPd2inkf0L960Gzg6ez2H/WvG/Bn7YRZyDgs8eB2wgtvY8xNaS+W7wS701eH9MJ8efEzxeDV4PIZYYlnXRnki3lAgklWQAJ7t7fftCM7sd+Ju7Xxhcb3+m3dt7O3xGQ7vnLXT+M9Lk+wfXuqrTnTp3nxUssf0E8EXgNmL7FRQAJ7h7k5ltAQZ2crwB33P3Ow+wXZFO6dKQpJInia3wCYCZtS0HnMf+pXsvi2P7K4hdkgL4x54qu3stse0rbzCzLGJxlgZJ4AxgUlC1Ghja7tAngCuC3g5mNs7MRvfT1yBpSIlAklWumRW3e1xP7JdqUTCA+gax5cMB/gv4npm9Snx7wdcB15vZOmKbj1T2dIC7v0psZdKFxPYrKDKz14B/Ija2gbvvAl4Ippve4u5PErv0tDyo+yAfTBQiB0TTR0X6SXCpp87d3cz+EVjo7hf0dJxI1DRGINJ/TgBuD2b67CGk7UBF+ko9AhGRNKcxAhGRNKdEICKS5pQIRETSnBKBiEiaUyIQEUlzSgQiImnu/wO1GI3Pgksi5gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_lr = 1e-2 * 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "escb = EarlyStoppingCallback(learn=learn, patience=5)\n",
    "smcb = SaveModelCallback(learn=learn)\n",
    "rpcb = ReduceLROnPlateauCallback(learn=learn, patience=3)\n",
    "sgcb = ShowGraph(learn=learn)\n",
    "csvcb = CSVLogger(learn=learn)\n",
    "\n",
    "callbacks = [escb, smcb, rpcb, sgcb, csvcb]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model\n",
    "\n",
    "Note: I don't actually do this in a notebook.  I execute training from a shell script at the root of this repository [run_train.sh](https://github.com/machine-learning-apps/IssuesLanguageModel/blob/master/run_train.sh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='184299' class='' max='395986', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      46.54% [184299/395986 14:40:30<16:51:20 3.4578]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(cyc_len=1,\n",
    "                    max_lr=1e-3,\n",
    "                    tot_epochs=10,\n",
    "                    callbacks=callbacks)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}