TXT_SentimentAnalysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TXT_SentimentAnalysis.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPr8XFiBB4tECg/c5t1RtGG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/btcnhung1299/tf-practice/blob/master/TXT_SentimentAnalysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkeH3OSbJAl-"
      },
      "source": [
        "import tensorflow_hub as hub\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "\n",
        "from tensorflow.keras.models import Sequential, Model\n",
        "from tensorflow.keras.layers import Dense, Embedding, Flatten, Bidirectional, LSTM, GRU, Dropout, GlobalAveragePooling1D, Conv1D"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "83w_fPzHL8t5"
      },
      "source": [
        "## Data gathering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUosNpchJhqM",
        "outputId": "21ec9505-3b3d-4a53-fb9b-d4ba4698798c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        }
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "tfds.disable_progress_bar()\n",
        "\n",
        "(ds_train, ds_val), ds_info = tfds.load(\"imdb_reviews\", split=[\"train\", \"test\"], shuffle_files=True, as_supervised=True, with_info=True)\n",
        "ds_info"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tfds.core.DatasetInfo(\n",
              "    name='imdb_reviews',\n",
              "    version=1.0.0,\n",
              "    description='Large Movie Review Dataset.\n",
              "This is a dataset for binary sentiment classification containing substantially more data than previous benchmark datasets. We provide a set of 25,000 highly polar movie reviews for training, and 25,000 for testing. There is additional unlabeled data for use as well.',\n",
              "    homepage='http://ai.stanford.edu/~amaas/data/sentiment/',\n",
              "    features=FeaturesDict({\n",
              "        'label': ClassLabel(shape=(), dtype=tf.int64, num_classes=2),\n",
              "        'text': Text(shape=(), dtype=tf.string),\n",
              "    }),\n",
              "    total_num_examples=100000,\n",
              "    splits={\n",
              "        'test': 25000,\n",
              "        'train': 25000,\n",
              "        'unsupervised': 50000,\n",
              "    },\n",
              "    supervised_keys=('text', 'label'),\n",
              "    citation=\"\"\"@InProceedings{maas-EtAl:2011:ACL-HLT2011,\n",
              "      author    = {Maas, Andrew L.  and  Daly, Raymond E.  and  Pham, Peter T.  and  Huang, Dan  and  Ng, Andrew Y.  and  Potts, Christopher},\n",
              "      title     = {Learning Word Vectors for Sentiment Analysis},\n",
              "      booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},\n",
              "      month     = {June},\n",
              "      year      = {2011},\n",
              "      address   = {Portland, Oregon, USA},\n",
              "      publisher = {Association for Computational Linguistics},\n",
              "      pages     = {142--150},\n",
              "      url       = {http://www.aclweb.org/anthology/P11-1015}\n",
              "    }\"\"\",\n",
              "    redistribution_info=,\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tWway6pGLdlE"
      },
      "source": [
        "**Inspect samples from datasets**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "__hGSUv4LeLQ"
      },
      "source": [
        "Binary classification:\n",
        "- Negative: 0\n",
        "- Positive: 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGcl41qVKuK1",
        "outputId": "5616d2b7-6c3d-4844-eadc-46bf97bcdfed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "sample = next(iter(ds_train))\n",
        "print(\"Review:\", sample[0].numpy())\n",
        "print(\"Label:\", sample[1].numpy())"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review: b\"This was an absolutely terrible movie. Don't be lured in by Christopher Walken or Michael Ironside. Both are great actors, but this must simply be their worst role in history. Even their great acting could not redeem this movie's ridiculous storyline. This movie is an early nineties US propaganda piece. The most pathetic scenes were those when the Columbian rebels were making their cases for revolutions. Maria Conchita Alonso appeared phony, and her pseudo-love affair with Walken was nothing but a pathetic emotional plug in a movie that was devoid of any real meaning. I am disappointed that there are movies like this, ruining actor's like Christopher Walken's good name. I could barely sit through it.\"\n",
            "Label: 0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SHKdpHyeMdcp",
        "outputId": "e10799f2-5c61-497f-afae-69ef23226027",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "train_data, train_labels = [], []\n",
        "for review, label in ds_train:\n",
        "  train_data.append(str(review.numpy()))\n",
        "  train_labels.append(label.numpy())\n",
        "\n",
        "val_data, val_labels = [], []\n",
        "for review, label in ds_val:\n",
        "  val_data.append(str(review.numpy()))\n",
        "  val_labels.append(label.numpy())\n",
        "\n",
        "print(\"Number of training samples:\", len(train_data))\n",
        "print(\"Number of validation samples:\", len(val_data))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training samples: 25000\n",
            "Number of validation samples: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LOXyAKXUx0bN"
      },
      "source": [
        "Brief analysis to decide proper hparams."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MqHWjrLiZxQk",
        "outputId": "1a81bf10-140e-49cf-f796-08c38a79c12d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        }
      },
      "source": [
        "pd.Series(train_labels).value_counts().plot(kind=\"bar\");"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD1CAYAAACyaJl6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAO2UlEQVR4nO3dbYxc5XnG8f9Vu6Z5aWMDK4vYTm0JN5FBrUJXhgqpqnCFDYliPiTIKCouteoPNW3SVkqg/WApBCmoVWlQA5UVuzERwlhuKqyE4FoGFFUthiUggnGIVxCwLV42sSFtUUhM7n7Yx82w2c16Z9Y7hv3/pNGccz/Pc+YeacXlOefMkKpCkjS7/Uq/G5Ak9Z9hIEkyDCRJhoEkCcNAkoRhIEkC5va7gW6de+65tXTp0n63IUlvK4899tgPqmpgbP1tGwZLly5laGio321I0ttKkufHq3uaSJJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJJ4G3/p7O1i6Q3f6HcL7xjf/8JH+t3CO4p/m9Pr7f736ScDSZJhIEkyDCRJGAaSJE4hDJJsS/JKkqc6an+X5LtJnkzyb0nmd4zdmGQ4yTNJVnfU17TacJIbOurLkuxv9XuSzJvONyhJmtypfDL4CrBmTG0vcGFV/TbwPeBGgCQrgHXABW3N7UnmJJkDfAm4AlgBXNPmAtwC3FpV5wPHgQ09vSNJ0pRNGgZV9S3g2Jjav1fVibb7MLC4ba8FdlTVG1X1HDAMrGyP4ap6tqp+AuwA1iYJcBmwq63fDlzV43uSJE3RdFwz+BPgm217EXC4Y+xIq01UPwd4tSNYTtYlSTOopzBI8rfACeCu6Wln0tfbmGQoydDIyMhMvKQkzQpdh0GSPwY+CnyyqqqVjwJLOqYtbrWJ6j8E5ieZO6Y+rqraUlWDVTU4MPAL/wtPSVKXugqDJGuAzwAfq6rXO4Z2A+uSnJVkGbAceAR4FFje7hyax+hF5t0tRB4EPt7Wrwfu7e6tSJK6dSq3lt4N/BfwwSRHkmwA/gn4dWBvkieS/DNAVR0AdgJPA/cDm6rqzXZN4HpgD3AQ2NnmAnwW+Kskw4xeQ9g6re9QkjSpSX+orqquGac84X+wq+pm4OZx6vcB941Tf5bRu40kSX3iN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSSJUwiDJNuSvJLkqY7a2Un2JjnUnhe0epLclmQ4yZNJLupYs77NP5RkfUf9d5N8p625LUmm+01Kkn65U/lk8BVgzZjaDcC+qloO7Gv7AFcAy9tjI3AHjIYHsBm4GFgJbD4ZIG3On3asG/takqTTbNIwqKpvAcfGlNcC29v2duCqjvqdNephYH6S84DVwN6qOlZVx4G9wJo29htV9XBVFXBnx7EkSTOk22sGC6vqxbb9ErCwbS8CDnfMO9Jqv6x+ZJy6JGkG9XwBuf2Lvqahl0kl2ZhkKMnQyMjITLykJM0K3YbBy+0UD+35lVY/CizpmLe41X5ZffE49XFV1ZaqGqyqwYGBgS5blySN1W0Y7AZO3hG0Hri3o35tu6voEuC1djppD3B5kgXtwvHlwJ429qMkl7S7iK7tOJYkaYbMnWxCkruBPwDOTXKE0buCvgDsTLIBeB64uk2/D7gSGAZeB64DqKpjSW4CHm3zPldVJy9K/xmjdyy9C/hme0iSZtCkYVBV10wwtGqcuQVsmuA424Bt49SHgAsn60OSdPr4DWRJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEmixzBI8pdJDiR5KsndSX4tybIk+5MMJ7knybw296y2P9zGl3Yc58ZWfybJ6t7ekiRpqroOgySLgL8ABqvqQmAOsA64Bbi1qs4HjgMb2pINwPFWv7XNI8mKtu4CYA1we5I53fYlSZq6Xk8TzQXelWQu8G7gReAyYFcb3w5c1bbXtn3a+KokafUdVfVGVT0HDAMre+xLkjQFXYdBVR0F/h54gdEQeA14DHi1qk60aUeARW17EXC4rT3R5p/TWR9njSRpBvRymmgBo/+qXwa8H3gPo6d5TpskG5MMJRkaGRk5nS8lSbNKL6eJ/hB4rqpGquqnwNeAS4H57bQRwGLgaNs+CiwBaOPvA37YWR9nzVtU1ZaqGqyqwYGBgR5alyR16iUMXgAuSfLudu5/FfA08CDw8TZnPXBv297d9mnjD1RVtfq6drfRMmA58EgPfUmSpmju5FPGV1X7k+wCvg2cAB4HtgDfAHYk+XyrbW1LtgJfTTIMHGP0DiKq6kCSnYwGyQlgU1W92W1fkqSp6zoMAKpqM7B5TPlZxrkbqKp+DHxiguPcDNzcSy+SpO75DWRJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEmixzBIMj/JriTfTXIwye8lOTvJ3iSH2vOCNjdJbksynOTJJBd1HGd9m38oyfpe35QkaWp6/WTwReD+qvoQ8DvAQeAGYF9VLQf2tX2AK4Dl7bERuAMgydnAZuBiYCWw+WSASJJmRtdhkOR9wO8DWwGq6idV9SqwFtjepm0Hrmrba4E7a9TDwPwk5wGrgb1VdayqjgN7gTXd9iVJmrpePhksA0aAf0nyeJIvJ3kPsLCqXmxzXgIWtu1FwOGO9UdabaK6JGmG9BIGc4GLgDuq6sPA//LzU0IAVFUB1cNrvEWSjUmGkgyNjIxM12EladbrJQyOAEeqan/b38VoOLzcTv/Qnl9p40eBJR3rF7faRPVfUFVbqmqwqgYHBgZ6aF2S1KnrMKiql4DDST7YSquAp4HdwMk7gtYD97bt3cC17a6iS4DX2umkPcDlSRa0C8eXt5okaYbM7XH9nwN3JZkHPAtcx2jA7EyyAXgeuLrNvQ+4EhgGXm9zqapjSW4CHm3zPldVx3rsS5I0BT2FQVU9AQyOM7RqnLkFbJrgONuAbb30Iknqnt9AliQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJKYhDJLMSfJ4kq+3/WVJ9icZTnJPknmtflbbH27jSzuOcWOrP5Nkda89SZKmZjo+GXwKONixfwtwa1WdDxwHNrT6BuB4q9/a5pFkBbAOuABYA9yeZM409CVJOkU9hUGSxcBHgC+3/QCXAbvalO3AVW17bdunja9q89cCO6rqjap6DhgGVvbSlyRpanr9ZPCPwGeAn7X9c4BXq+pE2z8CLGrbi4DDAG38tTb//+vjrJEkzYCuwyDJR4FXquqxaexnstfcmGQoydDIyMhMvawkveP18sngUuBjSb4P7GD09NAXgflJ5rY5i4GjbfsosASgjb8P+GFnfZw1b1FVW6pqsKoGBwYGemhdktSp6zCoqhuranFVLWX0AvADVfVJ4EHg423aeuDetr277dPGH6iqavV17W6jZcBy4JFu+5IkTd3cyadM2WeBHUk+DzwObG31rcBXkwwDxxgNEKrqQJKdwNPACWBTVb15GvqSJE1gWsKgqh4CHmrbzzLO3UBV9WPgExOsvxm4eTp6kSRNnd9AliQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRI9hEGSJUkeTPJ0kgNJPtXqZyfZm+RQe17Q6klyW5LhJE8muajjWOvb/ENJ1vf+tiRJU9HLJ4MTwF9X1QrgEmBTkhXADcC+qloO7Gv7AFcAy9tjI3AHjIYHsBm4GFgJbD4ZIJKkmdF1GFTVi1X17bb938BBYBGwFtjepm0Hrmrba4E7a9TDwPwk5wGrgb1VdayqjgN7gTXd9iVJmrppuWaQZCnwYWA/sLCqXmxDLwEL2/Yi4HDHsiOtNlF9vNfZmGQoydDIyMh0tC5JYhrCIMl7gX8FPl1VP+ocq6oCqtfX6DjelqoarKrBgYGB6TqsJM16PYVBkl9lNAjuqqqvtfLL7fQP7fmVVj8KLOlYvrjVJqpLkmZIL3cTBdgKHKyqf+gY2g2cvCNoPXBvR/3adlfRJcBr7XTSHuDyJAvahePLW02SNEPm9rD2UuCPgO8keaLV/gb4ArAzyQbgeeDqNnYfcCUwDLwOXAdQVceS3AQ82uZ9rqqO9dCXJGmKug6DqvoPIBMMrxpnfgGbJjjWNmBbt71IknrjN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSSJMygMkqxJ8kyS4SQ39LsfSZpNzogwSDIH+BJwBbACuCbJiv52JUmzxxkRBsBKYLiqnq2qnwA7gLV97kmSZo25/W6gWQQc7tg/Alw8dlKSjcDGtvs/SZ6Zgd5mg3OBH/S7icnkln53oD7x73N6/eZ4xTMlDE5JVW0BtvS7j3eaJENVNdjvPqTx+Pc5M86U00RHgSUd+4tbTZI0A86UMHgUWJ5kWZJ5wDpgd597kqRZ44w4TVRVJ5JcD+wB5gDbqupAn9uaTTz1pjOZf58zIFXV7x4kSX12ppwmkiT1kWEgSTIMJElnyAVkSQJI8iFGf31gUSsdBXZX1cH+dTU7+MlAb5Hkun73oNkpyWcZ/SmaAI+0R4C7/fHK08+7ifQWSV6oqg/0uw/NPkm+B1xQVT8dU58HHKiq5f3pbHbwNNEslOTJiYaAhTPZi9ThZ8D7gefH1M9rYzqNDIPZaSGwGjg+ph7gP2e+HQmATwP7khzi5z9c+QHgfOD6vnU1SxgGs9PXgfdW1RNjB5I8NPPtSFBV9yf5LUZ/0r7zAvKjVfVm/zqbHbxmIEnybiJJkmEgScIwkCRhGEiSMAwkScD/Ae3LBTGfEsb6AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NfqmdC5bvGoB",
        "outputId": "e9034a08-6b79-42e4-c16c-90afc1c88f34",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "pd.Series(train_data).apply(lambda x : len(x.split())).describe()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    25000.000000\n",
              "mean       233.776720\n",
              "std        173.715418\n",
              "min         10.000000\n",
              "25%        127.000000\n",
              "50%        174.000000\n",
              "75%        284.000000\n",
              "max       2470.000000\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1u0QqaZ6UP2w"
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "BUFFER_SIZE = 1024\n",
        "VOCAB_SIZE = 1000\n",
        "EMBED_DIM = 64\n",
        "MAX_SEQ_LEN = 256"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QI2TxhBYMLMr"
      },
      "source": [
        "## Featurizing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2urNVhgReOiV"
      },
      "source": [
        "**Tokenizing and padding sequences**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B0kIxgs8PHEw"
      },
      "source": [
        "The average number of tokens persample isi 233. Hence, we choose `MAX_SEQ_LEN = 256`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SQOgBDDzMWMs"
      },
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "tokenizer = Tokenizer(num_words=VOCAB_SIZE, oov_token=\"<OOV>\")\n",
        "tokenizer.fit_on_texts(train_data)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_rK_OdbN5PG"
      },
      "source": [
        "def create_tfds(tokenizer, X, y, padding=False):\n",
        "  if padding:\n",
        "    X = pad_sequences(tokenizer.texts_to_sequences(X), maxlen=MAX_SEQ_LEN, padding=\"post\")\n",
        "  return tf.data.Dataset.from_tensor_slices((X, y)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
        "\n",
        "ds_train = create_tfds(tokenizer, train_data, train_labels, padding=True)\n",
        "ds_val = create_tfds(tokenizer, val_data, val_labels, padding=True)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7IPHXwlCb25h"
      },
      "source": [
        "## Model Architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h1rDLOHCcKP8",
        "outputId": "93461f99-3e2c-4921-9b73-602dfc4926ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=VOCAB_SIZE, input_length=MAX_SEQ_LEN, output_dim=EMBED_DIM))\n",
        "model.add(Conv1D(filters=64, kernel_size=5, activation=\"swish\"))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(GlobalAveragePooling1D())\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(1, activation=\"sigmoid\"))\n",
        "model.summary()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 256, 64)           64000     \n",
            "_________________________________________________________________\n",
            "conv1d (Conv1D)              (None, 252, 64)           20544     \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 252, 64)           0         \n",
            "_________________________________________________________________\n",
            "global_average_pooling1d (Gl (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 65        \n",
            "=================================================================\n",
            "Total params: 84,609\n",
            "Trainable params: 84,609\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A4PWtiSpckcJ"
      },
      "source": [
        "model.compile(optimizer=\"rmsprop\", loss=\"binary_crossentropy\", metrics=[\"acc\"])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqzRLLcMcvM_",
        "outputId": "add85ac9-b308-4a40-b6c4-2c1810b6243a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 833
        }
      },
      "source": [
        "callbacks = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=4)\n",
        "model.fit(ds_train, epochs=50, validation_data=ds_val, callbacks=[callbacks]);"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "391/391 [==============================] - 4s 10ms/step - loss: 0.4924 - acc: 0.7622 - val_loss: 0.3519 - val_acc: 0.8513\n",
            "Epoch 2/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.3475 - acc: 0.8509 - val_loss: 0.3442 - val_acc: 0.8491\n",
            "Epoch 3/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.3329 - acc: 0.8601 - val_loss: 0.3254 - val_acc: 0.8612\n",
            "Epoch 4/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.3263 - acc: 0.8619 - val_loss: 0.3406 - val_acc: 0.8516\n",
            "Epoch 5/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.3181 - acc: 0.8668 - val_loss: 0.3224 - val_acc: 0.8607\n",
            "Epoch 6/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.3108 - acc: 0.8709 - val_loss: 0.3179 - val_acc: 0.8644\n",
            "Epoch 7/50\n",
            "391/391 [==============================] - 4s 9ms/step - loss: 0.3035 - acc: 0.8739 - val_loss: 0.3293 - val_acc: 0.8590\n",
            "Epoch 8/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2967 - acc: 0.8756 - val_loss: 0.3111 - val_acc: 0.8678\n",
            "Epoch 9/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2937 - acc: 0.8780 - val_loss: 0.3116 - val_acc: 0.8674\n",
            "Epoch 10/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2900 - acc: 0.8800 - val_loss: 0.3141 - val_acc: 0.8661\n",
            "Epoch 11/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2864 - acc: 0.8811 - val_loss: 0.3204 - val_acc: 0.8649\n",
            "Epoch 12/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2847 - acc: 0.8825 - val_loss: 0.3086 - val_acc: 0.8694\n",
            "Epoch 13/50\n",
            "391/391 [==============================] - 3s 8ms/step - loss: 0.2832 - acc: 0.8836 - val_loss: 0.3084 - val_acc: 0.8694\n",
            "Epoch 14/50\n",
            "391/391 [==============================] - 3s 8ms/step - loss: 0.2805 - acc: 0.8845 - val_loss: 0.3088 - val_acc: 0.8690\n",
            "Epoch 15/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2797 - acc: 0.8852 - val_loss: 0.3091 - val_acc: 0.8688\n",
            "Epoch 16/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2762 - acc: 0.8865 - val_loss: 0.3090 - val_acc: 0.8693\n",
            "Epoch 17/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2742 - acc: 0.8863 - val_loss: 0.3081 - val_acc: 0.8695\n",
            "Epoch 18/50\n",
            "391/391 [==============================] - 3s 8ms/step - loss: 0.2721 - acc: 0.8905 - val_loss: 0.3080 - val_acc: 0.8696\n",
            "Epoch 19/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2700 - acc: 0.8902 - val_loss: 0.3091 - val_acc: 0.8700\n",
            "Epoch 20/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2678 - acc: 0.8892 - val_loss: 0.3077 - val_acc: 0.8708\n",
            "Epoch 21/50\n",
            "391/391 [==============================] - 3s 8ms/step - loss: 0.2658 - acc: 0.8923 - val_loss: 0.3102 - val_acc: 0.8703\n",
            "Epoch 22/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2642 - acc: 0.8926 - val_loss: 0.3117 - val_acc: 0.8686\n",
            "Epoch 23/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2615 - acc: 0.8924 - val_loss: 0.3163 - val_acc: 0.8679\n",
            "Epoch 24/50\n",
            "391/391 [==============================] - 3s 9ms/step - loss: 0.2602 - acc: 0.8932 - val_loss: 0.3100 - val_acc: 0.8711\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o_ZHnYI8ncpl"
      },
      "source": [
        "### Transfer learning w. pretrained embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "atVA7Hx6AFrd"
      },
      "source": [
        "ds_train = create_tfds(tokenizer, train_data, train_labels, padding=False)\n",
        "ds_val = create_tfds(tokenizer, val_data, val_labels, padding=False)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Smp7hTc4SR9R",
        "outputId": "fb013f1a-b45d-45e0-cc18-a929813584f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "embedding_url = \"https://tfhub.dev/google/nnlm-en-dim50-with-normalization/2\"\n",
        "\n",
        "model = Sequential()\n",
        "model.add(hub.KerasLayer(embedding_url, input_shape=(), dtype=tf.string, trainable=True))\n",
        "model.add(Dropout(0.4))\n",
        "model.add(Dense(1, activation=\"sigmoid\"))\n",
        "model.summary()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 5 calls to <function recreate_function.<locals>.restored_function_body at 0x7f18e8322048> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 5 calls to <function recreate_function.<locals>.restored_function_body at 0x7f18e8322048> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:6 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f18e83229d8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:6 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f18e83229d8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "keras_layer_2 (KerasLayer)   (None, 50)                48190600  \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 50)                0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1)                 51        \n",
            "=================================================================\n",
            "Total params: 48,190,651\n",
            "Trainable params: 48,190,651\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7VZVufLvRqcW"
      },
      "source": [
        "model.compile(optimizer=\"rmsprop\", loss=\"binary_crossentropy\", metrics=[\"acc\"])"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GbkeVW4tZmqr",
        "outputId": "2eb381a4-bbfb-4b4a-a410-ff3e46dc0aa7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "callbacks = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=4)\n",
        "model.fit(ds_train, epochs=20, validation_data=ds_val, callbacks=[callbacks]);"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "391/391 [==============================] - 8s 20ms/step - loss: 0.4962 - acc: 0.7715 - val_loss: 0.3565 - val_acc: 0.8586\n",
            "Epoch 2/20\n",
            "391/391 [==============================] - 8s 20ms/step - loss: 0.2940 - acc: 0.8820 - val_loss: 0.2865 - val_acc: 0.8827\n",
            "Epoch 3/20\n",
            "391/391 [==============================] - 8s 20ms/step - loss: 0.2335 - acc: 0.9086 - val_loss: 0.2654 - val_acc: 0.8918\n",
            "Epoch 4/20\n",
            "391/391 [==============================] - 8s 20ms/step - loss: 0.2002 - acc: 0.9234 - val_loss: 0.2600 - val_acc: 0.8932\n",
            "Epoch 5/20\n",
            "391/391 [==============================] - 8s 19ms/step - loss: 0.1794 - acc: 0.9330 - val_loss: 0.2612 - val_acc: 0.8938\n",
            "Epoch 6/20\n",
            "391/391 [==============================] - 8s 19ms/step - loss: 0.1633 - acc: 0.9391 - val_loss: 0.2650 - val_acc: 0.8928\n",
            "Epoch 7/20\n",
            "391/391 [==============================] - 8s 19ms/step - loss: 0.1481 - acc: 0.9446 - val_loss: 0.2714 - val_acc: 0.8910\n",
            "Epoch 8/20\n",
            "391/391 [==============================] - 8s 19ms/step - loss: 0.1385 - acc: 0.9482 - val_loss: 0.2792 - val_acc: 0.8887\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uzWkFnn8QG92",
        "outputId": "919c2fd8-e47c-4f4a-d517-67c57283a8be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "model.evaluate(ds_val)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "391/391 [==============================] - 2s 6ms/step - loss: 0.2792 - acc: 0.8887\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.2792399525642395, 0.8886799812316895]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    }
  ]
}