Sentiment Analysis v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep Sentiment Analysis Tutorial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NC9qqvWoJYZT"
   },
   "source": [
    "### Setup Environment\n",
    "- On Google Colab make sure you select Python 3/GPU runtime before running the code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bnCJOwMIJhoE"
   },
   "source": [
    "#### Choose Python 3 + GPU/CPU\n",
    "\n",
    "<img src=\"https://i.stack.imgur.com/khwGc.png\" width=\"400\"></img>\n",
    "<img src=\"https://i.stack.imgur.com/5iL6w.png\" width=\"400\"></img>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=0\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Skip Download\r\n"
     ]
    }
   ],
   "source": [
    "![ ! -d data ] && mkdir data/\n",
    "![ -f data/aclImdb_v1.tar.gz ] && echo \"Skip Download\"\n",
    "![ ! -f data/aclImdb_v1.tar.gz ] && wget -N https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz -P data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data already extracted\n",
      "CPU times: user 6.34 ms, sys: 4.91 ms, total: 11.2 ms\n",
      "Wall time: 224 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "![ -d data/aclImdb/ ] && echo \"Data already extracted\"\n",
    "![ ! -d data/aclImdb/ ] && tar -xzf data/aclImdb_v1.tar.gz -C data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import nltk\n",
    "from collections import Counter\n",
    "from tqdm import tqdm_notebook\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import seq2seq\n",
    "from tensorflow.contrib.rnn import DropoutWrapper\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/bishal/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQ_LEN = 50\n",
    "BATCH_SIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Lang:\n",
    "    def __init__(self, counter, vocab_size):\n",
    "        self.word2id = {}\n",
    "        self.id2word = {}\n",
    "        self.pad = \"<PAD>\"\n",
    "        self.sos = \"<SOS>\"\n",
    "        self.eos = \"<EOS>\"\n",
    "        self.unk = \"<UNK>\"\n",
    "        \n",
    "        self.ipad = 0\n",
    "        self.isos = 1\n",
    "        self.ieos = 2\n",
    "        self.iunk = 3\n",
    "        \n",
    "        self.word2id[self.pad] = 0\n",
    "        self.word2id[self.sos] = 1\n",
    "        self.word2id[self.eos] = 2\n",
    "        self.word2id[self.unk] = 3\n",
    "        \n",
    "        self.id2word[0] = self.pad\n",
    "        self.id2word[1] = self.sos\n",
    "        self.id2word[2] = self.eos\n",
    "        self.id2word[3] = self.unk\n",
    "        \n",
    "        curr_id = 4\n",
    "        for w, c in counter.most_common(vocab_size-curr_id):\n",
    "            self.word2id[w] = curr_id\n",
    "            self.id2word[curr_id] = w\n",
    "            curr_id += 1\n",
    "    \n",
    "    def encodeSentence(self, wseq, max_len=-1):\n",
    "        # wseq = nltk.tokenize.word_tokenize(s.lower().strip())\n",
    "        if max_len == -1:\n",
    "            return [self.word2id[w] if w in self.word2id else self.iunk for w in wseq]\n",
    "        else:\n",
    "            return ([self.word2id[w] if w in self.word2id else self.iunk for w in wseq] + [self.ieos] + [self.ipad]*max_len)[:max_len]\n",
    "        \n",
    "    def encodeSentence2(self, wseq, max_len=-1):\n",
    "        # wseq = nltk.tokenize.word_tokenize(s.lower().strip()) \n",
    "        return min(max_len, len(wseq)+1), \\\n",
    "            ([self.word2id[w] if w in self.word2id else self.iunk for w in wseq] + \\\n",
    "                [self.ieos] + [self.ipad]*max_len)[:max_len]\n",
    "    \n",
    "    def decodeSentence(self, id_seq):\n",
    "        id_seq = np.array(id_seq + [self.ieos])\n",
    "        j = np.argmax(id_seq==self.ieos)\n",
    "        s = ' '.join([self.id2word[x] for x in id_seq[:j]])\n",
    "        s = s.replace(self.unk, \"UNK\")\n",
    "        return s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YTsqaGfEKyku"
   },
   "source": [
    "### Let's read in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SJtY9mw6Kykv"
   },
   "outputs": [],
   "source": [
    "data_folder = 'data/aclImdb/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mJOwy_iyKyky"
   },
   "outputs": [],
   "source": [
    "rp = os.path.join(data_folder, 'train/pos')\n",
    "train_positive = [os.path.join(rp, f) for f in os.listdir(rp)]\n",
    "rp = os.path.join(data_folder, 'train/neg')\n",
    "train_negative = [os.path.join(rp, f) for f in os.listdir(rp)]\n",
    "\n",
    "rp = os.path.join(data_folder, 'test/pos')\n",
    "test_positive = [os.path.join(rp, f) for f in os.listdir(rp)]\n",
    "rp = os.path.join(data_folder, 'test/neg')\n",
    "test_negative = [os.path.join(rp, f) for f in os.listdir(rp)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Pmq06cnLKyk7"
   },
   "source": [
    "#### Limit number of samples\n",
    "To quickly train a small model, consider setting n_train and n_test to some relatively small numbers e.g. `1000`. Set, \n",
    "`n_train = n_test = -1` to use all the samples available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-kTiqMYYKyk8"
   },
   "outputs": [],
   "source": [
    "n_train = 100000\n",
    "n_test = 2500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TTY671XQKyk4"
   },
   "outputs": [],
   "source": [
    "re_html_cleaner = re.compile(r\"<.*?>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5340c356f6a4d5cbc99bfd40eda1fa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Crunching +ve samples: ', max=12500, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb259e1c81dc4d88aa5da62fac2eada9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Crunching -ve samples: ', max=12500, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "en_counter = Counter()\n",
    "train_data = []\n",
    "for _fname in tqdm_notebook(train_positive[:n_train], desc=\"Crunching +ve samples: \"):\n",
    "    with open(_fname) as f:\n",
    "        text = f.read().strip()\n",
    "        text = re_html_cleaner.sub(\" \", text)\n",
    "        wseq = nltk.tokenize.word_tokenize(text.lower())\n",
    "        en_counter += Counter(wseq)\n",
    "        train_data.append((wseq, 1))\n",
    "        \n",
    "for _fname in tqdm_notebook(train_negative[:n_train], desc=\"Crunching -ve samples: \"):\n",
    "    with open(_fname) as f:\n",
    "        text = f.read().strip()\n",
    "        text = re_html_cleaner.sub(\" \", text)\n",
    "        wseq = nltk.tokenize.word_tokenize(text.lower())\n",
    "        en_counter += Counter(wseq)\n",
    "        train_data.append((wseq, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c05a3e73f744813a712e247731034ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Crunching +ve samples: ', max=2500, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "784004e010874b84bd7c2df2a2974851",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Crunching -ve samples: ', max=2500, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_data = []\n",
    "for _fname in tqdm_notebook(test_positive[:n_test], desc=\"Crunching +ve samples: \"):\n",
    "    with open(_fname) as f:\n",
    "        text = f.read().strip()\n",
    "        text = re_html_cleaner.sub(\" \", text)\n",
    "        wseq = nltk.tokenize.word_tokenize(text.lower())\n",
    "        test_data.append((wseq, 1))\n",
    "        \n",
    "for _fname in tqdm_notebook(test_negative[:n_test], desc=\"Crunching -ve samples: \"):\n",
    "    with open(_fname) as f:\n",
    "        text = f.read().strip()\n",
    "        text = re_html_cleaner.sub(\" \", text)\n",
    "        wseq = nltk.tokenize.word_tokenize(text.lower())\n",
    "        test_data.append((wseq, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Most common en words in dataset:\n",
      " [('the', 334752), (',', 275881), ('.', 271448), ('and', 163327), ('a', 162162), ('of', 145428), ('to', 135195), ('is', 110396), ('it', 95772), ('in', 93249)]\n",
      "\n",
      "Total (en)words gathered from dataset: 105920\n"
     ]
    }
   ],
   "source": [
    "# A few sample english words\n",
    "print(\"\\nMost common en words in dataset:\\n\", en_counter.most_common(10))\n",
    "\n",
    "print(\"\\nTotal (en)words gathered from dataset:\", len(en_counter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "V = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_lang = Lang(en_counter, V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test en encoding: [131, 33, 27, 182, 58]\n",
      "Test en decoding: where are you going ?\n"
     ]
    }
   ],
   "source": [
    "wseq = nltk.tokenize.word_tokenize(\"Where are you going?\".lower())\n",
    "print(\"Test en encoding:\", en_lang.encodeSentence(wseq))\n",
    "print(\"Test en decoding:\", en_lang.decodeSentence(en_lang.encodeSentence(wseq, 10)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The RNN based Sentence Classifier architecture\n",
    "- We will implement a RNN based classifier architecture for sentiment analysis in Tensorflow r1.13.1 / r1.14\n",
    "- Debugging Tip: Always keep track of tensor dimensions!\n",
    "- **Tensorflow Computation Graph** - We will build a tf computation graph first. This is the representation used by tf for any neural network architecture. Once the computation graph is built, you can feed data to it for training or inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Word Embedding Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/bishal/miniconda3/envs/tf_gpu/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "en_word_emb_matrix = tf.get_variable(\"en_word_emb_matrix\", (V, 300), dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Placeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "keep_prob = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = tf.placeholder(tf.int32, (None, MAX_SEQ_LEN))\n",
    "input_lens = tf.placeholder(tf.int32, (None, ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_placeholder = tf.placeholder(tf.int32, (None,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tensorflow Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_emb = tf.nn.embedding_lookup(en_word_emb_matrix, input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(50), Dimension(300)])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_emb.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### RNN Units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-25-090b5be7a571>:2: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    }
   ],
   "source": [
    "# Create a single GRU cell\n",
    "encoder_cell = tf.nn.rnn_cell.GRUCell(128)\n",
    "# Add dropout : Dropout is applied to the hidden state output at every time step\n",
    "encoder_cell = DropoutWrapper(encoder_cell, output_keep_prob=keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-26-48ec6b10a5e9>:5: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/bishal/miniconda3/envs/tf_gpu/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /home/bishal/miniconda3/envs/tf_gpu/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "# Unrolling of time-sequence\n",
    "# Apply the encoder cell on input sequence and unroll computation upto\n",
    "# max sequence length\n",
    "enc_outputs, enc_state = tf.nn.dynamic_rnn(\n",
    "    encoder_cell, input_emb, sequence_length=input_lens, initial_state=encoder_cell.zero_state(BATCH_SIZE, dtype=tf.float32)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(64), Dimension(50), Dimension(128)])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc_outputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(64), Dimension(128)])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc_state.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classifier Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple fully connected linear layer\n",
    "# W^T*X + b\n",
    "dense_layer = tf.layers.Dense(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Approaches:\n",
    "As input to the final linear layers use mean of the hidden states?\n",
    "\n",
    "or\n",
    "\n",
    "As input to the final linear layers use the last hidden state?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Approch 1: Take mean of enc_outputs across dimension 1\n",
    "- **IMPORTANT:** Need to **mask** the positions in input sentence that doesn't contain any inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# masks = tf.sequence_mask(input_lens, MAX_SEQ_LEN, dtype=tf.float32, name='masks')\n",
    "# class_prob = tf.nn.sigmoid(\n",
    "#                 dense_layer(\n",
    "#                     tf.reduce_mean(\n",
    "#                         enc_outputs*masks[:, :, None], 1)\n",
    "#                 )\n",
    "# ) \n",
    "\n",
    "# print(class_prob.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Approch 2: Use enc_state (final hidden state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 1)\n"
     ]
    }
   ],
   "source": [
    "class_prob = tf.nn.sigmoid(dense_layer(enc_state))\n",
    "print(class_prob.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loss and Optimizers [softmax_cross_entropy]\n",
    "Note that `onehot_labels` and `logits` must have the same shape, e.g. `[batch_size, num_classes]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?,)\n",
      "(64, 1)\n"
     ]
    }
   ],
   "source": [
    "print(y_placeholder.shape)\n",
    "print(class_prob.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loss function - softmax cross entropy\n",
    "y_ = tf.cast(y_placeholder[:, None], dtype=tf.float32)\n",
    "cost = -y_*tf.log(class_prob + 1e-12) - (1-y_)*tf.log(1-class_prob + 1e-12)\n",
    "cost = tf.reduce_mean(cost)\n",
    "\n",
    "# Optimizer\n",
    "optimizer = tf.train.AdamOptimizer(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_op = optimizer.minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tensorflow Sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess_config = tf.ConfigProto()\n",
    "sess_config.gpu_options.allow_growth = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession(config=sess_config)\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Minibatch Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(41)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_n = len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_n = len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def small_test():\n",
    "    all_true = []\n",
    "    all_preds = []\n",
    "    for m in range(0, test_n, BATCH_SIZE):\n",
    "        n = m + BATCH_SIZE\n",
    "        if n > test_n:\n",
    "            break\n",
    "\n",
    "        input_batch = np.zeros((BATCH_SIZE, MAX_SEQ_LEN), dtype=np.int32)\n",
    "        input_lens_batch = np.zeros((BATCH_SIZE,), dtype=np.int32)\n",
    "        true_class_batch = np.zeros((BATCH_SIZE))\n",
    "        for i in range(m, n):\n",
    "            b,a = en_lang.encodeSentence2(test_data[i][0], MAX_SEQ_LEN)\n",
    "            input_batch[i-m,:] = a\n",
    "            input_lens_batch[i-m] = b\n",
    "            true_class_batch[i-m] = test_data[i][1]\n",
    "\n",
    "        feed_dict={\n",
    "            input_ids: input_batch,\n",
    "            input_lens: input_lens_batch,\n",
    "            keep_prob: 1.0\n",
    "        }\n",
    "        pred_batch = sess.run(class_prob, feed_dict=feed_dict)\n",
    "        # acc = accuracy_score(true_class_batch, pred_batch > 0.5)\n",
    "        all_true.extend(list(true_class_batch))\n",
    "        all_preds.extend(list(pred_batch[:,0]))\n",
    "    \n",
    "    all_true = np.array(all_true)\n",
    "    all_preds = np.array(all_preds)\n",
    "    prec = precision_score(all_true, all_preds > 0.5)*100\n",
    "    rec = recall_score(all_true, all_preds > 0.5)*100\n",
    "    f1 = f1_score(all_true, all_preds > 0.5)*100\n",
    "    print(f\"Precision: {prec:2.2F}, Recall: {rec:2.2F}, F1-Score: {f1:2.2F}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e79e5aa02d54487f9ccdb795afc29444",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 50.30, Recall: 88.64, F1-Score: 64.18\n",
      "Precision: 50.77, Recall: 91.28, F1-Score: 65.25\n",
      "Precision: 54.95, Recall: 83.52, F1-Score: 66.29\n",
      "Precision: 57.33, Recall: 91.72, F1-Score: 70.55\n",
      "Precision: 77.05, Recall: 37.20, F1-Score: 50.18\n",
      "Precision: 65.37, Recall: 82.24, F1-Score: 72.84\n",
      "Precision: 69.77, Recall: 73.20, F1-Score: 71.44\n",
      "Precision: 65.84, Recall: 84.48, F1-Score: 74.00\n",
      "Precision: 73.20, Recall: 69.36, F1-Score: 71.23\n",
      "Precision: 69.46, Recall: 79.16, F1-Score: 74.00\n",
      "Precision: 75.91, Recall: 68.44, F1-Score: 71.98\n",
      "Precision: 76.59, Recall: 66.76, F1-Score: 71.34\n",
      "Precision: 71.31, Recall: 79.24, F1-Score: 75.07\n",
      "Precision: 76.40, Recall: 68.64, F1-Score: 72.31\n",
      "Precision: 73.93, Recall: 74.96, F1-Score: 74.44\n",
      "Precision: 81.44, Recall: 50.56, F1-Score: 62.39\n",
      "Precision: 72.21, Recall: 78.68, F1-Score: 75.31\n",
      "Precision: 78.34, Recall: 66.56, F1-Score: 71.97\n",
      "Precision: 70.41, Recall: 83.20, F1-Score: 76.27\n",
      "Precision: 73.46, Recall: 75.60, F1-Score: 74.51\n",
      "Precision: 71.12, Recall: 81.96, F1-Score: 76.16\n",
      "Precision: 76.98, Recall: 67.80, F1-Score: 72.10\n",
      "Precision: 73.32, Recall: 78.04, F1-Score: 75.61\n",
      "Precision: 76.89, Recall: 69.20, F1-Score: 72.84\n",
      "Precision: 73.14, Recall: 78.20, F1-Score: 75.58\n",
      "Precision: 77.40, Recall: 69.20, F1-Score: 73.07\n",
      "Precision: 72.97, Recall: 79.28, F1-Score: 76.00\n",
      "Precision: 75.68, Recall: 75.20, F1-Score: 75.44\n",
      "Precision: 71.29, Recall: 82.84, F1-Score: 76.63\n",
      "Precision: 75.41, Recall: 76.04, F1-Score: 75.72\n",
      "Precision: 72.99, Recall: 80.00, F1-Score: 76.34\n",
      "Precision: 78.59, Recall: 68.72, F1-Score: 73.32\n",
      "Precision: 73.31, Recall: 81.64, F1-Score: 77.25\n",
      "Precision: 75.93, Recall: 76.08, F1-Score: 76.00\n",
      "Precision: 74.75, Recall: 78.28, F1-Score: 76.48\n",
      "Precision: 77.18, Recall: 73.20, F1-Score: 75.14\n",
      "Precision: 74.51, Recall: 77.88, F1-Score: 76.16\n",
      "Precision: 77.06, Recall: 73.76, F1-Score: 75.37\n",
      "Precision: 76.90, Recall: 75.76, F1-Score: 76.32\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9c0ec132dc04b198f4cfef4af9398c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 73.85, Recall: 80.08, F1-Score: 76.84\n",
      "Precision: 76.18, Recall: 74.32, F1-Score: 75.24\n",
      "Precision: 77.32, Recall: 70.76, F1-Score: 73.89\n",
      "Precision: 76.12, Recall: 75.12, F1-Score: 75.62\n",
      "Precision: 74.19, Recall: 78.76, F1-Score: 76.41\n",
      "Precision: 76.54, Recall: 72.04, F1-Score: 74.22\n",
      "Precision: 78.19, Recall: 67.40, F1-Score: 72.40\n",
      "Precision: 71.75, Recall: 80.68, F1-Score: 75.96\n",
      "Precision: 75.86, Recall: 73.28, F1-Score: 74.55\n",
      "Precision: 71.11, Recall: 81.64, F1-Score: 76.01\n",
      "Precision: 77.24, Recall: 68.28, F1-Score: 72.48\n",
      "Precision: 72.17, Recall: 79.16, F1-Score: 75.51\n",
      "Precision: 73.09, Recall: 77.68, F1-Score: 75.32\n",
      "Precision: 75.79, Recall: 72.52, F1-Score: 74.12\n",
      "Precision: 73.05, Recall: 78.92, F1-Score: 75.87\n",
      "Precision: 77.82, Recall: 69.88, F1-Score: 73.64\n",
      "Precision: 77.57, Recall: 69.32, F1-Score: 73.22\n",
      "Precision: 75.01, Recall: 75.64, F1-Score: 75.32\n",
      "Precision: 73.56, Recall: 79.00, F1-Score: 76.18\n",
      "Precision: 79.96, Recall: 65.92, F1-Score: 72.26\n",
      "Precision: 75.16, Recall: 76.00, F1-Score: 75.58\n",
      "Precision: 75.87, Recall: 70.16, F1-Score: 72.90\n",
      "Precision: 75.62, Recall: 75.68, F1-Score: 75.65\n",
      "Precision: 74.14, Recall: 76.60, F1-Score: 75.35\n",
      "Precision: 75.21, Recall: 74.40, F1-Score: 74.80\n",
      "Precision: 74.12, Recall: 75.72, F1-Score: 74.91\n",
      "Precision: 78.93, Recall: 68.16, F1-Score: 73.15\n",
      "Precision: 75.42, Recall: 76.36, F1-Score: 75.89\n",
      "Precision: 78.60, Recall: 68.16, F1-Score: 73.01\n",
      "Precision: 70.98, Recall: 83.64, F1-Score: 76.79\n",
      "Precision: 76.70, Recall: 72.68, F1-Score: 74.64\n",
      "Precision: 75.38, Recall: 76.56, F1-Score: 75.97\n",
      "Precision: 77.42, Recall: 72.84, F1-Score: 75.06\n",
      "Precision: 77.10, Recall: 74.32, F1-Score: 75.68\n",
      "Precision: 72.71, Recall: 82.28, F1-Score: 77.20\n",
      "Precision: 78.98, Recall: 72.00, F1-Score: 75.33\n",
      "Precision: 76.05, Recall: 78.12, F1-Score: 77.07\n",
      "Precision: 79.82, Recall: 69.60, F1-Score: 74.36\n",
      "Precision: 74.51, Recall: 80.32, F1-Score: 77.31\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "332480513eda4507b981f6a1e14f7987",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 78.82, Recall: 73.52, F1-Score: 76.08\n",
      "Precision: 78.27, Recall: 74.04, F1-Score: 76.09\n",
      "Precision: 76.93, Recall: 75.08, F1-Score: 75.99\n",
      "Precision: 75.60, Recall: 75.84, F1-Score: 75.72\n",
      "Precision: 77.53, Recall: 72.60, F1-Score: 74.98\n",
      "Precision: 75.00, Recall: 77.76, F1-Score: 76.36\n",
      "Precision: 75.66, Recall: 75.48, F1-Score: 75.57\n",
      "Precision: 75.95, Recall: 73.52, F1-Score: 74.72\n",
      "Precision: 75.36, Recall: 76.32, F1-Score: 75.83\n",
      "Precision: 74.70, Recall: 76.64, F1-Score: 75.66\n",
      "Precision: 73.71, Recall: 78.40, F1-Score: 75.98\n",
      "Precision: 77.25, Recall: 71.72, F1-Score: 74.38\n",
      "Precision: 74.83, Recall: 76.96, F1-Score: 75.88\n",
      "Precision: 78.61, Recall: 63.52, F1-Score: 70.27\n",
      "Precision: 73.21, Recall: 79.04, F1-Score: 76.01\n",
      "Precision: 77.04, Recall: 69.92, F1-Score: 73.31\n",
      "Precision: 76.99, Recall: 69.48, F1-Score: 73.04\n",
      "Precision: 70.86, Recall: 81.12, F1-Score: 75.64\n",
      "Precision: 77.74, Recall: 68.04, F1-Score: 72.57\n",
      "Precision: 75.74, Recall: 74.44, F1-Score: 75.09\n",
      "Precision: 77.92, Recall: 66.64, F1-Score: 71.84\n",
      "Precision: 74.74, Recall: 76.12, F1-Score: 75.43\n",
      "Precision: 75.37, Recall: 73.56, F1-Score: 74.45\n",
      "Precision: 74.12, Recall: 73.68, F1-Score: 73.90\n",
      "Precision: 77.69, Recall: 69.36, F1-Score: 73.29\n",
      "Precision: 73.14, Recall: 79.40, F1-Score: 76.14\n",
      "Precision: 77.88, Recall: 67.60, F1-Score: 72.38\n",
      "Precision: 72.26, Recall: 79.00, F1-Score: 75.48\n",
      "Precision: 76.74, Recall: 70.88, F1-Score: 73.70\n",
      "Precision: 75.76, Recall: 74.24, F1-Score: 74.99\n",
      "Precision: 77.57, Recall: 71.24, F1-Score: 74.27\n",
      "Precision: 73.77, Recall: 77.08, F1-Score: 75.39\n",
      "Precision: 78.16, Recall: 69.00, F1-Score: 73.30\n",
      "Precision: 76.03, Recall: 74.60, F1-Score: 75.31\n",
      "Precision: 76.62, Recall: 73.92, F1-Score: 75.24\n",
      "Precision: 73.05, Recall: 80.00, F1-Score: 76.37\n",
      "Precision: 77.89, Recall: 70.44, F1-Score: 73.98\n",
      "Precision: 76.10, Recall: 74.88, F1-Score: 75.48\n",
      "Precision: 75.57, Recall: 75.60, F1-Score: 75.58\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2af7c3df28cc44a88ea232a00f24938c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 76.94, Recall: 73.40, F1-Score: 75.13\n",
      "Precision: 75.76, Recall: 75.88, F1-Score: 75.82\n",
      "Precision: 77.94, Recall: 68.96, F1-Score: 73.17\n",
      "Precision: 75.16, Recall: 73.60, F1-Score: 74.37\n",
      "Precision: 76.28, Recall: 70.48, F1-Score: 73.26\n",
      "Precision: 74.08, Recall: 76.04, F1-Score: 75.05\n",
      "Precision: 74.90, Recall: 73.52, F1-Score: 74.20\n",
      "Precision: 75.78, Recall: 72.48, F1-Score: 74.10\n",
      "Precision: 76.13, Recall: 71.20, F1-Score: 73.58\n",
      "Precision: 77.20, Recall: 69.76, F1-Score: 73.29\n",
      "Precision: 72.74, Recall: 76.84, F1-Score: 74.73\n",
      "Precision: 77.73, Recall: 68.12, F1-Score: 72.61\n",
      "Precision: 74.61, Recall: 73.68, F1-Score: 74.14\n",
      "Precision: 74.86, Recall: 73.00, F1-Score: 73.92\n",
      "Precision: 76.04, Recall: 71.08, F1-Score: 73.48\n",
      "Precision: 71.58, Recall: 76.08, F1-Score: 73.76\n",
      "Precision: 76.07, Recall: 67.76, F1-Score: 71.67\n",
      "Precision: 75.19, Recall: 68.60, F1-Score: 71.74\n",
      "Precision: 72.29, Recall: 76.48, F1-Score: 74.32\n",
      "Precision: 75.02, Recall: 72.20, F1-Score: 73.58\n",
      "Precision: 75.19, Recall: 71.04, F1-Score: 73.06\n",
      "Precision: 75.18, Recall: 70.52, F1-Score: 72.78\n",
      "Precision: 73.99, Recall: 72.04, F1-Score: 73.00\n",
      "Precision: 75.04, Recall: 72.40, F1-Score: 73.70\n",
      "Precision: 76.27, Recall: 70.84, F1-Score: 73.45\n",
      "Precision: 73.60, Recall: 71.04, F1-Score: 72.30\n",
      "Precision: 74.45, Recall: 71.44, F1-Score: 72.91\n",
      "Precision: 76.00, Recall: 69.92, F1-Score: 72.83\n",
      "Precision: 73.33, Recall: 75.00, F1-Score: 74.15\n",
      "Precision: 76.44, Recall: 67.88, F1-Score: 71.91\n",
      "Precision: 73.16, Recall: 76.64, F1-Score: 74.86\n",
      "Precision: 75.59, Recall: 69.00, F1-Score: 72.15\n",
      "Precision: 73.65, Recall: 72.32, F1-Score: 72.98\n",
      "Precision: 75.92, Recall: 70.64, F1-Score: 73.19\n",
      "Precision: 74.31, Recall: 69.76, F1-Score: 71.96\n",
      "Precision: 73.20, Recall: 74.40, F1-Score: 73.79\n",
      "Precision: 76.99, Recall: 66.80, F1-Score: 71.54\n",
      "Precision: 73.49, Recall: 76.08, F1-Score: 74.76\n",
      "Precision: 75.51, Recall: 72.28, F1-Score: 73.86\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f040bbd02bc43e69826dbbfb0fbd36d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 74.07, Recall: 73.68, F1-Score: 73.87\n",
      "Precision: 76.00, Recall: 70.04, F1-Score: 72.90\n",
      "Precision: 74.89, Recall: 71.00, F1-Score: 72.90\n",
      "Precision: 74.88, Recall: 70.00, F1-Score: 72.36\n",
      "Precision: 73.78, Recall: 73.16, F1-Score: 73.47\n",
      "Precision: 74.02, Recall: 72.60, F1-Score: 73.30\n",
      "Precision: 74.49, Recall: 71.96, F1-Score: 73.20\n",
      "Precision: 76.06, Recall: 69.52, F1-Score: 72.64\n",
      "Precision: 75.21, Recall: 71.96, F1-Score: 73.55\n",
      "Precision: 74.38, Recall: 73.64, F1-Score: 74.01\n",
      "Precision: 75.12, Recall: 70.16, F1-Score: 72.55\n",
      "Precision: 73.68, Recall: 73.44, F1-Score: 73.56\n",
      "Precision: 75.44, Recall: 72.00, F1-Score: 73.68\n",
      "Precision: 75.23, Recall: 71.56, F1-Score: 73.35\n",
      "Precision: 74.17, Recall: 71.08, F1-Score: 72.59\n",
      "Precision: 73.92, Recall: 70.64, F1-Score: 72.24\n",
      "Precision: 72.26, Recall: 75.96, F1-Score: 74.06\n",
      "Precision: 75.45, Recall: 66.52, F1-Score: 70.71\n",
      "Precision: 73.34, Recall: 74.92, F1-Score: 74.12\n",
      "Precision: 74.24, Recall: 72.76, F1-Score: 73.49\n",
      "Precision: 72.34, Recall: 70.84, F1-Score: 71.58\n",
      "Precision: 74.41, Recall: 69.44, F1-Score: 71.84\n",
      "Precision: 75.09, Recall: 68.24, F1-Score: 71.50\n",
      "Precision: 73.47, Recall: 71.32, F1-Score: 72.38\n",
      "Precision: 74.46, Recall: 68.32, F1-Score: 71.26\n",
      "Precision: 73.42, Recall: 73.04, F1-Score: 73.23\n",
      "Precision: 75.26, Recall: 67.28, F1-Score: 71.05\n",
      "Precision: 71.51, Recall: 74.68, F1-Score: 73.06\n",
      "Precision: 74.29, Recall: 68.20, F1-Score: 71.12\n",
      "Precision: 72.99, Recall: 71.88, F1-Score: 72.43\n",
      "Precision: 74.33, Recall: 68.92, F1-Score: 71.52\n",
      "Precision: 74.53, Recall: 69.52, F1-Score: 71.94\n",
      "Precision: 73.06, Recall: 72.12, F1-Score: 72.58\n",
      "Precision: 73.93, Recall: 69.08, F1-Score: 71.42\n",
      "Precision: 73.36, Recall: 70.72, F1-Score: 72.02\n",
      "Precision: 72.53, Recall: 73.20, F1-Score: 72.86\n",
      "Precision: 74.83, Recall: 68.84, F1-Score: 71.71\n",
      "Precision: 73.45, Recall: 70.92, F1-Score: 72.16\n",
      "Precision: 73.89, Recall: 67.80, F1-Score: 70.71\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "009e87f36923493e855de545800ad578",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 75.27, Recall: 68.80, F1-Score: 71.89\n",
      "Precision: 73.59, Recall: 72.68, F1-Score: 73.13\n",
      "Precision: 71.99, Recall: 76.28, F1-Score: 74.07\n",
      "Precision: 76.03, Recall: 67.12, F1-Score: 71.30\n",
      "Precision: 73.98, Recall: 73.12, F1-Score: 73.55\n",
      "Precision: 75.33, Recall: 69.24, F1-Score: 72.16\n",
      "Precision: 72.51, Recall: 74.92, F1-Score: 73.70\n",
      "Precision: 74.05, Recall: 70.76, F1-Score: 72.37\n",
      "Precision: 74.12, Recall: 70.56, F1-Score: 72.30\n",
      "Precision: 73.52, Recall: 73.28, F1-Score: 73.40\n",
      "Precision: 72.05, Recall: 73.92, F1-Score: 72.97\n",
      "Precision: 73.67, Recall: 69.60, F1-Score: 71.58\n",
      "Precision: 73.33, Recall: 71.48, F1-Score: 72.39\n",
      "Precision: 72.89, Recall: 74.96, F1-Score: 73.91\n",
      "Precision: 74.76, Recall: 67.40, F1-Score: 70.89\n",
      "Precision: 74.44, Recall: 68.96, F1-Score: 71.59\n",
      "Precision: 72.62, Recall: 73.72, F1-Score: 73.16\n",
      "Precision: 75.66, Recall: 65.04, F1-Score: 69.95\n",
      "Precision: 71.67, Recall: 74.88, F1-Score: 73.24\n",
      "Precision: 73.46, Recall: 70.40, F1-Score: 71.90\n",
      "Precision: 74.49, Recall: 67.52, F1-Score: 70.84\n",
      "Precision: 73.60, Recall: 71.92, F1-Score: 72.75\n",
      "Precision: 74.46, Recall: 70.56, F1-Score: 72.46\n",
      "Precision: 73.79, Recall: 68.92, F1-Score: 71.27\n",
      "Precision: 72.58, Recall: 71.88, F1-Score: 72.23\n",
      "Precision: 74.09, Recall: 70.12, F1-Score: 72.05\n",
      "Precision: 71.63, Recall: 76.36, F1-Score: 73.92\n",
      "Precision: 74.24, Recall: 68.48, F1-Score: 71.24\n",
      "Precision: 73.63, Recall: 69.80, F1-Score: 71.66\n",
      "Precision: 74.90, Recall: 68.04, F1-Score: 71.31\n",
      "Precision: 73.36, Recall: 72.60, F1-Score: 72.98\n",
      "Precision: 73.54, Recall: 69.80, F1-Score: 71.62\n",
      "Precision: 73.83, Recall: 71.20, F1-Score: 72.49\n",
      "Precision: 74.93, Recall: 66.00, F1-Score: 70.18\n",
      "Precision: 72.59, Recall: 74.16, F1-Score: 73.37\n",
      "Precision: 76.40, Recall: 64.60, F1-Score: 70.00\n",
      "Precision: 71.90, Recall: 73.60, F1-Score: 72.74\n",
      "Precision: 74.96, Recall: 66.68, F1-Score: 70.58\n",
      "Precision: 74.99, Recall: 69.08, F1-Score: 71.91\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a0b3550289f42a1b8107949c73e3861",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 75.17, Recall: 70.72, F1-Score: 72.88\n",
      "Precision: 75.11, Recall: 69.88, F1-Score: 72.40\n",
      "Precision: 73.39, Recall: 73.80, F1-Score: 73.59\n",
      "Precision: 75.49, Recall: 67.88, F1-Score: 71.48\n",
      "Precision: 75.10, Recall: 69.48, F1-Score: 72.18\n",
      "Precision: 74.35, Recall: 68.76, F1-Score: 71.45\n",
      "Precision: 74.46, Recall: 69.28, F1-Score: 71.78\n",
      "Precision: 74.86, Recall: 69.20, F1-Score: 71.92\n",
      "Precision: 74.05, Recall: 71.12, F1-Score: 72.56\n",
      "Precision: 73.49, Recall: 71.52, F1-Score: 72.49\n",
      "Precision: 76.28, Recall: 64.96, F1-Score: 70.17\n",
      "Precision: 74.38, Recall: 68.76, F1-Score: 71.46\n",
      "Precision: 73.44, Recall: 72.44, F1-Score: 72.94\n",
      "Precision: 74.82, Recall: 67.76, F1-Score: 71.12\n",
      "Precision: 73.69, Recall: 69.92, F1-Score: 71.76\n",
      "Precision: 74.03, Recall: 70.92, F1-Score: 72.44\n",
      "Precision: 73.96, Recall: 69.52, F1-Score: 71.67\n",
      "Precision: 73.51, Recall: 69.92, F1-Score: 71.67\n",
      "Precision: 76.27, Recall: 66.08, F1-Score: 70.81\n",
      "Precision: 74.91, Recall: 69.88, F1-Score: 72.31\n",
      "Precision: 74.54, Recall: 70.04, F1-Score: 72.22\n",
      "Precision: 72.13, Recall: 76.40, F1-Score: 74.20\n",
      "Precision: 74.63, Recall: 69.76, F1-Score: 72.11\n",
      "Precision: 74.53, Recall: 68.60, F1-Score: 71.44\n",
      "Precision: 73.35, Recall: 72.00, F1-Score: 72.67\n",
      "Precision: 74.00, Recall: 69.32, F1-Score: 71.58\n",
      "Precision: 73.95, Recall: 70.40, F1-Score: 72.13\n",
      "Precision: 73.71, Recall: 70.64, F1-Score: 72.14\n",
      "Precision: 73.17, Recall: 71.24, F1-Score: 72.19\n",
      "Precision: 72.95, Recall: 69.80, F1-Score: 71.34\n",
      "Precision: 73.96, Recall: 68.84, F1-Score: 71.31\n",
      "Precision: 73.22, Recall: 70.88, F1-Score: 72.03\n",
      "Precision: 73.35, Recall: 70.12, F1-Score: 71.70\n",
      "Precision: 74.76, Recall: 67.76, F1-Score: 71.09\n",
      "Precision: 71.26, Recall: 73.00, F1-Score: 72.12\n",
      "Precision: 75.77, Recall: 68.56, F1-Score: 71.99\n",
      "Precision: 74.76, Recall: 67.64, F1-Score: 71.02\n",
      "Precision: 73.17, Recall: 70.92, F1-Score: 72.03\n",
      "Precision: 73.09, Recall: 72.48, F1-Score: 72.79\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7a65cf0c7184cbd8a57054fb7bd2454",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 72.90, Recall: 71.44, F1-Score: 72.16\n",
      "Precision: 74.49, Recall: 66.92, F1-Score: 70.50\n",
      "Precision: 73.27, Recall: 71.80, F1-Score: 72.53\n",
      "Precision: 72.73, Recall: 70.40, F1-Score: 71.54\n",
      "Precision: 72.13, Recall: 64.80, F1-Score: 68.27\n",
      "Precision: 73.15, Recall: 66.48, F1-Score: 69.66\n",
      "Precision: 71.95, Recall: 71.52, F1-Score: 71.74\n",
      "Precision: 73.68, Recall: 69.76, F1-Score: 71.67\n",
      "Precision: 74.36, Recall: 68.68, F1-Score: 71.41\n",
      "Precision: 73.17, Recall: 71.88, F1-Score: 72.52\n",
      "Precision: 73.57, Recall: 70.48, F1-Score: 71.99\n",
      "Precision: 73.46, Recall: 69.08, F1-Score: 71.20\n",
      "Precision: 73.38, Recall: 70.12, F1-Score: 71.71\n",
      "Precision: 72.91, Recall: 71.28, F1-Score: 72.09\n",
      "Precision: 72.20, Recall: 71.28, F1-Score: 71.74\n",
      "Precision: 71.33, Recall: 73.76, F1-Score: 72.53\n",
      "Precision: 74.00, Recall: 69.00, F1-Score: 71.41\n",
      "Precision: 74.83, Recall: 65.88, F1-Score: 70.07\n",
      "Precision: 73.84, Recall: 69.68, F1-Score: 71.70\n",
      "Precision: 74.70, Recall: 67.44, F1-Score: 70.89\n",
      "Precision: 74.22, Recall: 69.32, F1-Score: 71.69\n",
      "Precision: 72.27, Recall: 73.72, F1-Score: 72.99\n",
      "Precision: 72.46, Recall: 73.88, F1-Score: 73.16\n",
      "Precision: 73.21, Recall: 70.72, F1-Score: 71.94\n",
      "Precision: 73.46, Recall: 70.96, F1-Score: 72.19\n",
      "Precision: 73.13, Recall: 71.08, F1-Score: 72.09\n",
      "Precision: 72.42, Recall: 70.88, F1-Score: 71.64\n",
      "Precision: 72.86, Recall: 70.00, F1-Score: 71.40\n",
      "Precision: 73.03, Recall: 70.72, F1-Score: 71.86\n",
      "Precision: 73.21, Recall: 70.60, F1-Score: 71.88\n",
      "Precision: 71.69, Recall: 72.52, F1-Score: 72.10\n",
      "Precision: 73.90, Recall: 67.72, F1-Score: 70.67\n",
      "Precision: 70.95, Recall: 72.60, F1-Score: 71.77\n",
      "Precision: 73.56, Recall: 69.00, F1-Score: 71.21\n",
      "Precision: 75.27, Recall: 64.28, F1-Score: 69.34\n",
      "Precision: 70.39, Recall: 75.68, F1-Score: 72.94\n",
      "Precision: 72.86, Recall: 69.92, F1-Score: 71.36\n",
      "Precision: 74.54, Recall: 66.16, F1-Score: 70.10\n",
      "Precision: 71.75, Recall: 72.84, F1-Score: 72.29\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d15dc9bb6f64c4a849a9cab3aa04b86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 72.21, Recall: 71.60, F1-Score: 71.90\n",
      "Precision: 74.76, Recall: 65.52, F1-Score: 69.84\n",
      "Precision: 74.14, Recall: 68.00, F1-Score: 70.94\n",
      "Precision: 72.53, Recall: 72.56, F1-Score: 72.55\n",
      "Precision: 73.70, Recall: 69.04, F1-Score: 71.29\n",
      "Precision: 75.89, Recall: 64.60, F1-Score: 69.79\n",
      "Precision: 72.12, Recall: 71.60, F1-Score: 71.86\n",
      "Precision: 72.12, Recall: 70.88, F1-Score: 71.49\n",
      "Precision: 73.51, Recall: 69.16, F1-Score: 71.27\n",
      "Precision: 74.82, Recall: 67.28, F1-Score: 70.85\n",
      "Precision: 71.88, Recall: 74.76, F1-Score: 73.29\n",
      "Precision: 73.70, Recall: 69.72, F1-Score: 71.65\n",
      "Precision: 72.82, Recall: 68.92, F1-Score: 70.82\n",
      "Precision: 71.65, Recall: 71.48, F1-Score: 71.57\n",
      "Precision: 72.22, Recall: 70.92, F1-Score: 71.56\n",
      "Precision: 73.43, Recall: 69.96, F1-Score: 71.65\n",
      "Precision: 73.27, Recall: 68.32, F1-Score: 70.71\n",
      "Precision: 69.70, Recall: 73.72, F1-Score: 71.66\n",
      "Precision: 74.37, Recall: 62.44, F1-Score: 67.88\n",
      "Precision: 72.62, Recall: 68.52, F1-Score: 70.51\n",
      "Precision: 72.08, Recall: 70.44, F1-Score: 71.25\n",
      "Precision: 72.49, Recall: 69.36, F1-Score: 70.89\n",
      "Precision: 71.81, Recall: 73.68, F1-Score: 72.73\n",
      "Precision: 72.15, Recall: 71.80, F1-Score: 71.97\n",
      "Precision: 73.49, Recall: 64.64, F1-Score: 68.78\n",
      "Precision: 71.70, Recall: 70.24, F1-Score: 70.96\n",
      "Precision: 71.73, Recall: 73.16, F1-Score: 72.44\n",
      "Precision: 73.09, Recall: 68.76, F1-Score: 70.86\n",
      "Precision: 72.47, Recall: 69.92, F1-Score: 71.17\n",
      "Precision: 71.82, Recall: 68.40, F1-Score: 70.07\n",
      "Precision: 72.22, Recall: 69.68, F1-Score: 70.93\n",
      "Precision: 73.05, Recall: 68.40, F1-Score: 70.65\n",
      "Precision: 71.37, Recall: 72.68, F1-Score: 72.02\n",
      "Precision: 74.02, Recall: 67.24, F1-Score: 70.47\n",
      "Precision: 74.03, Recall: 65.92, F1-Score: 69.74\n",
      "Precision: 71.16, Recall: 71.84, F1-Score: 71.50\n",
      "Precision: 72.48, Recall: 69.64, F1-Score: 71.03\n",
      "Precision: 73.74, Recall: 68.08, F1-Score: 70.80\n",
      "Precision: 72.38, Recall: 71.16, F1-Score: 71.76\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b83c7ff33f74ec59729d62465a16020",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=391), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 74.58, Recall: 65.36, F1-Score: 69.67\n",
      "Precision: 71.66, Recall: 71.32, F1-Score: 71.49\n",
      "Precision: 71.91, Recall: 70.44, F1-Score: 71.17\n",
      "Precision: 74.42, Recall: 64.60, F1-Score: 69.16\n",
      "Precision: 74.16, Recall: 66.92, F1-Score: 70.35\n",
      "Precision: 72.85, Recall: 70.20, F1-Score: 71.50\n",
      "Precision: 71.93, Recall: 72.16, F1-Score: 72.04\n",
      "Precision: 73.02, Recall: 68.08, F1-Score: 70.46\n",
      "Precision: 71.28, Recall: 72.28, F1-Score: 71.78\n",
      "Precision: 72.09, Recall: 69.52, F1-Score: 70.78\n",
      "Precision: 73.93, Recall: 67.28, F1-Score: 70.45\n",
      "Precision: 73.56, Recall: 67.88, F1-Score: 70.61\n",
      "Precision: 72.76, Recall: 68.92, F1-Score: 70.79\n",
      "Precision: 71.49, Recall: 71.00, F1-Score: 71.24\n",
      "Precision: 72.00, Recall: 68.92, F1-Score: 70.43\n",
      "Precision: 72.26, Recall: 70.64, F1-Score: 71.44\n",
      "Precision: 72.83, Recall: 70.24, F1-Score: 71.51\n",
      "Precision: 72.42, Recall: 68.28, F1-Score: 70.29\n",
      "Precision: 71.27, Recall: 70.84, F1-Score: 71.05\n",
      "Precision: 71.23, Recall: 71.20, F1-Score: 71.21\n",
      "Precision: 72.73, Recall: 69.12, F1-Score: 70.88\n",
      "Precision: 73.54, Recall: 68.80, F1-Score: 71.09\n",
      "Precision: 72.34, Recall: 69.68, F1-Score: 70.99\n",
      "Precision: 72.26, Recall: 69.60, F1-Score: 70.90\n",
      "Precision: 72.80, Recall: 68.84, F1-Score: 70.76\n",
      "Precision: 71.85, Recall: 71.88, F1-Score: 71.87\n",
      "Precision: 71.85, Recall: 70.96, F1-Score: 71.40\n",
      "Precision: 74.28, Recall: 65.60, F1-Score: 69.67\n",
      "Precision: 74.00, Recall: 67.04, F1-Score: 70.35\n",
      "Precision: 72.96, Recall: 70.04, F1-Score: 71.47\n",
      "Precision: 72.52, Recall: 70.72, F1-Score: 71.61\n",
      "Precision: 72.18, Recall: 70.68, F1-Score: 71.42\n",
      "Precision: 72.75, Recall: 68.88, F1-Score: 70.76\n",
      "Precision: 73.30, Recall: 67.20, F1-Score: 70.12\n",
      "Precision: 71.78, Recall: 69.80, F1-Score: 70.78\n",
      "Precision: 71.07, Recall: 71.92, F1-Score: 71.49\n",
      "Precision: 72.60, Recall: 69.52, F1-Score: 71.03\n",
      "Precision: 72.88, Recall: 68.92, F1-Score: 70.85\n",
      "Precision: 72.15, Recall: 68.80, F1-Score: 70.43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for _e in range(5):\n",
    "    # Mix things up a bit.\n",
    "    random.shuffle(train_data)\n",
    "    pbar = tqdm_notebook(range(0, train_n, BATCH_SIZE))\n",
    "    batch_loss = 0\n",
    "    bxi = 0\n",
    "    for m in pbar:\n",
    "        n = m + BATCH_SIZE\n",
    "        if n <= train_n:\n",
    "            # print(\"Epoch Complete... \\n\")\n",
    "\n",
    "            input_batch = np.zeros((BATCH_SIZE, MAX_SEQ_LEN), dtype=np.int32)\n",
    "            input_lens_batch = np.zeros((BATCH_SIZE,), dtype=np.int32)\n",
    "            true_class_batch = np.zeros((BATCH_SIZE))\n",
    "            for i in range(m, n):\n",
    "                b,a = en_lang.encodeSentence2(train_data[i][0], MAX_SEQ_LEN)\n",
    "                input_batch[i-m,:] = a\n",
    "                input_lens_batch[i-m] = b\n",
    "                true_class_batch[i-m] = train_data[i][1]\n",
    "\n",
    "            feed_dict={\n",
    "                input_ids: input_batch,\n",
    "                input_lens: input_lens_batch,\n",
    "                y_placeholder: true_class_batch,\n",
    "                keep_prob: 0.6\n",
    "            }\n",
    "            sess.run(train_op, feed_dict=feed_dict)\n",
    "            batch_loss += sess.run(cost, feed_dict=feed_dict)\n",
    "            pbar.set_description(f\"Epoch: {_e} >> Loss: {batch_loss/(bxi+1):2.2F}:\")\n",
    "            bxi += 1\n",
    "            if (1 + n//BATCH_SIZE) % 10 == 0:\n",
    "                small_test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Improving Further\n",
    "- This was a very simple RNN based model for the task.\n",
    "- You can still improve it a lot by tweaking hyperparameters e.g.\n",
    " - lstm size \n",
    " - dropout\n",
    " - learning rate \n",
    "- or modifying the architecture e.g.\n",
    " - Add bidirectional RNNs\n",
    " - Use multiple layers of RNN cells\n",
    " - Add more hidden layers to the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (tf_gpu)",
   "language": "python",
   "name": "tf_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "320px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}