TXT_SarcasmDetection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TXT_SarcasmDetection.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyM59cp1Jav+DPaQCZJvu+Vc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/btcnhung1299/tf-practice/blob/master/TXT_SarcasmDetection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZdRgGWsm9En"
      },
      "source": [
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TkU9l69oo1q1"
      },
      "source": [
        "## Data Gathering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vs-76pZTmsRh",
        "outputId": "2b80993d-08ce-4b23-bdbe-74073a5d4e05",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "!wget https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-09-29 12:49:35--  https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.142.128, 173.194.203.128, 74.125.195.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.142.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 5643545 (5.4M) [application/json]\n",
            "Saving to: ‘sarcasm.json.4’\n",
            "\n",
            "\rsarcasm.json.4        0%[                    ]       0  --.-KB/s               \rsarcasm.json.4      100%[===================>]   5.38M  --.-KB/s    in 0.03s   \n",
            "\n",
            "2020-09-29 12:49:35 (214 MB/s) - ‘sarcasm.json.4’ saved [5643545/5643545]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vj9sbCXcnJFg",
        "outputId": "645ce4a3-9c14-4474-bbb2-cb7d1cafff67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df = pd.read_json(\"./sarcasm.json\")\n",
        "df.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>article_link</th>\n",
              "      <th>headline</th>\n",
              "      <th>is_sarcastic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://www.huffingtonpost.com/entry/versace-b...</td>\n",
              "      <td>former versace store clerk sues over secret 'b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://www.huffingtonpost.com/entry/roseanne-...</td>\n",
              "      <td>the 'roseanne' revival catches up to our thorn...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://local.theonion.com/mom-starting-to-fea...</td>\n",
              "      <td>mom starting to fear son's web series closest ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://politics.theonion.com/boehner-just-wan...</td>\n",
              "      <td>boehner just wants wife to listen, not come up...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www.huffingtonpost.com/entry/jk-rowlin...</td>\n",
              "      <td>j.k. rowling wishes snape happy birthday in th...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                        article_link  ... is_sarcastic\n",
              "0  https://www.huffingtonpost.com/entry/versace-b...  ...            0\n",
              "1  https://www.huffingtonpost.com/entry/roseanne-...  ...            0\n",
              "2  https://local.theonion.com/mom-starting-to-fea...  ...            1\n",
              "3  https://politics.theonion.com/boehner-just-wan...  ...            1\n",
              "4  https://www.huffingtonpost.com/entry/jk-rowlin...  ...            0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qbc4Ti0LXe9z",
        "outputId": "973af1eb-e96a-4bf5-d16f-08c77ab36447",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        }
      },
      "source": [
        "df[\"is_sarcastic\"].value_counts().plot(kind=\"bar\");"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD1CAYAAACyaJl6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQZklEQVR4nO3df6zddX3H8edr7WD+iLbIXYdt2W1Gpylki3gDXUwWI0tb1Fj+UAMxo2ONzWLddDFRcH80AUkwW8YkUZJOOstiqIS50CjaNYgxy1bgIgoUxN7xq7cBuXoLbiP+KL73x/l0Hq/3cnvPudxTep+P5OR+v+/P5/P9vk/S9NXz/X7PbaoKSdLi9huDbkCSNHiGgSTJMJAkGQaSJAwDSRKGgSQJWDroBnp15pln1vDw8KDbkKRXlPvuu++HVTU0tf6KDYPh4WFGR0cH3YYkvaIkeXK6upeJJEmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJIlX8JfOXimGr/zqoFs4ZTxx3bsG3YJ0yvKTgSTJMJAkGQaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSOIEwSLIrybNJHppm7GNJKsmZbT9JbkgyluSBJOd3zd2S5FB7bemqvzXJg23NDUkyX29OknRiTuSTwReATVOLSVYDG4CnusoXA2vbaxtwY5t7BrADuBC4ANiRZHlbcyPwwa51v3YuSdLLa9YwqKpvAZPTDF0PfByortpm4ObqOAAsS3IWsBHYX1WTVXUU2A9samOvq6oDVVXAzcAl/b0lSdJc9XTPIMlm4EhVfXfK0ErgcNf+eKu9VH18mrokaQHN+beWJnk18Ek6l4gWVJJtdC4/cfbZZy/06SXplNXLJ4PfA9YA303yBLAK+HaS3wGOAKu75q5qtZeqr5qmPq2q2llVI1U1MjQ01EPrkqTpzDkMqurBqvrtqhquqmE6l3bOr6pngL3A5e2povXA81X1NLAP2JBkebtxvAHY18Z+nGR9e4rocuD2eXpvkqQTdCKPlt4C/CfwpiTjSba+xPQ7gMeAMeAfgQ8BVNUkcA1wb3td3Wq0OZ9va/4L+Fpvb0WS1KtZ7xlU1WWzjA93bRewfYZ5u4Bd09RHgfNm60OS9PLxG8iSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjiBMEiyK8mzSR7qqv1tku8leSDJvyZZ1jV2VZKxJI8m2dhV39RqY0mu7KqvSXJ3q38pyWnz+QYlSbM7kU8GXwA2TantB86rqj8Avg9cBZBkHXApcG5b87kkS5IsAT4LXAysAy5rcwE+DVxfVecAR4Gtfb0jSdKczRoGVfUtYHJK7d+q6ljbPQCsatubgT1V9dOqehwYAy5or7GqeqyqfgbsATYnCfAO4La2fjdwSZ/vSZI0R/Nxz+DPga+17ZXA4a6x8Vabqf4G4LmuYDlelyQtoL7CIMnfAMeAL85PO7Oeb1uS0SSjExMTC3FKSVoUeg6DJH8GvBv4QFVVKx8BVndNW9VqM9V/BCxLsnRKfVpVtbOqRqpqZGhoqNfWJUlT9BQGSTYBHwfeU1UvdA3tBS5NcnqSNcBa4B7gXmBte3LoNDo3mfe2ELkLeG9bvwW4vbe3Iknq1dLZJiS5BXg7cGaScWAHnaeHTgf2d+4Bc6Cq/qKqDia5FXiYzuWj7VX1YjvOh4F9wBJgV1UdbKf4BLAnyaeA+4Gb5vH9SZrB8JVfHXQLp5QnrnvXoFvoy6xhUFWXTVOe8S/sqroWuHaa+h3AHdPUH6PztJEkaUD8BrIkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJnEAYJNmV5NkkD3XVzkiyP8mh9nN5qyfJDUnGkjyQ5PyuNVva/ENJtnTV35rkwbbmhrT/VFmStHBO5JPBF4BNU2pXAndW1VrgzrYPcDGwtr22ATdCJzyAHcCFdP6/4x3HA6TN+WDXuqnnkiS9zGYNg6r6FjA5pbwZ2N22dwOXdNVvro4DwLIkZwEbgf1VNVlVR4H9wKY29rqqOlBVBdzcdSxJ0gLp9Z7Biqp6um0/A6xo2yuBw13zxlvtperj09QlSQuo7xvI7V/0NQ+9zCrJtiSjSUYnJiYW4pSStCj0GgY/aJd4aD+fbfUjwOqueata7aXqq6apT6uqdlbVSFWNDA0N9di6JGmqXsNgL3D8iaAtwO1d9cvbU0Xrgefb5aR9wIYky9uN4w3Avjb24yTr21NEl3cdS5K0QJbONiHJLcDbgTOTjNN5Kug64NYkW4Engfe36XcA7wTGgBeAKwCqajLJNcC9bd7VVXX8pvSH6Dyx9Crga+0lSVpAs4ZBVV02w9BF08wtYPsMx9kF7JqmPgqcN1sfkqSXj99AliQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJJEn2GQ5K+THEzyUJJbkvxWkjVJ7k4yluRLSU5rc09v+2NtfLjrOFe1+qNJNvb3liRJc9VzGCRZCfwVMFJV5wFLgEuBTwPXV9U5wFFga1uyFTja6te3eSRZ19adC2wCPpdkSa99SZLmrt/LREuBVyVZCrwaeBp4B3BbG98NXNK2N7d92vhFSdLqe6rqp1X1ODAGXNBnX5KkOeg5DKrqCPB3wFN0QuB54D7guao61qaNAyvb9krgcFt7rM1/Q3d9mjWSpAXQz2Wi5XT+Vb8GeCPwGjqXeV42SbYlGU0yOjEx8XKeSpIWlX4uE/0J8HhVTVTVz4EvA28DlrXLRgCrgCNt+wiwGqCNvx74UXd9mjW/oqp2VtVIVY0MDQ310bokqVs/YfAUsD7Jq9u1/4uAh4G7gPe2OVuA29v23rZPG/9GVVWrX9qeNloDrAXu6aMvSdIcLZ19yvSq6u4ktwHfBo4B9wM7ga8Ce5J8qtVuaktuAv45yRgwSecJIqrqYJJb6QTJMWB7Vb3Ya1+SpLnrOQwAqmoHsGNK+TGmeRqoqn4CvG+G41wLXNtPL5Kk3vkNZEmSYSBJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ9BkGSZYluS3J95I8kuSPkpyRZH+SQ+3n8jY3SW5IMpbkgSTndx1nS5t/KMmWft+UJGlu+v1k8Bng61X1ZuAPgUeAK4E7q2otcGfbB7gYWNte24AbAZKcAewALgQuAHYcDxBJ0sLoOQySvB74Y+AmgKr6WVU9B2wGdrdpu4FL2vZm4ObqOAAsS3IWsBHYX1WTVXUU2A9s6rUvSdLc9fPJYA0wAfxTkvuTfD7Ja4AVVfV0m/MMsKJtrwQOd60fb7WZ6pKkBdJPGCwFzgdurKq3AP/LLy8JAVBVBVQf5/gVSbYlGU0yOjExMV+HlaRFr58wGAfGq+rutn8bnXD4Qbv8Q/v5bBs/AqzuWr+q1Waq/5qq2llVI1U1MjQ01EfrkqRuPYdBVT0DHE7ypla6CHgY2AscfyJoC3B7294LXN6eKloPPN8uJ+0DNiRZ3m4cb2g1SdICWdrn+r8EvpjkNOAx4Ao6AXNrkq3Ak8D729w7gHcCY8ALbS5VNZnkGuDeNu/qqprssy9J0hz0FQZV9R1gZJqhi6aZW8D2GY6zC9jVTy+SpN75DWRJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSmIcwSLIkyf1JvtL21yS5O8lYki+1/x+ZJKe3/bE2Ptx1jKta/dEkG/vtSZI0N/PxyeAjwCNd+58Grq+qc4CjwNZW3wocbfXr2zySrAMuBc4FNgGfS7JkHvqSJJ2gvsIgySrgXcDn236AdwC3tSm7gUva9ua2Txu/qM3fDOypqp9W1ePAGHBBP31Jkuam308G/wB8HPhF238D8FxVHWv748DKtr0SOAzQxp9v8/+/Ps0aSdIC6DkMkrwbeLaq7pvHfmY757Yko0lGJyYmFuq0knTK6+eTwduA9yR5AthD5/LQZ4BlSZa2OauAI237CLAaoI2/HvhRd32aNb+iqnZW1UhVjQwNDfXRuiSpW89hUFVXVdWqqhqmcwP4G1X1AeAu4L1t2hbg9ra9t+3Txr9RVdXql7anjdYAa4F7eu1LkjR3S2efMmefAPYk+RRwP3BTq98E/HOSMWCSToBQVQeT3Ao8DBwDtlfViy9DX5KkGcxLGFTVN4Fvtu3HmOZpoKr6CfC+GdZfC1w7H71IkubObyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSaKPMEiyOsldSR5OcjDJR1r9jCT7kxxqP5e3epLckGQsyQNJzu861pY2/1CSLf2/LUnSXPTzyeAY8LGqWgesB7YnWQdcCdxZVWuBO9s+wMXA2vbaBtwInfAAdgAXAhcAO44HiCRpYfQcBlX1dFV9u23/N/AIsBLYDOxu03YDl7TtzcDN1XEAWJbkLGAjsL+qJqvqKLAf2NRrX5KkuZuXewZJhoG3AHcDK6rq6Tb0DLCiba8EDnctG2+1meqSpAXSdxgkeS3wL8BHq+rH3WNVVUD1e46uc21LMppkdGJiYr4OK0mLXl9hkOQ36QTBF6vqy638g3b5h/bz2VY/AqzuWr6q1Waq/5qq2llVI1U1MjQ01E/rkqQu/TxNFOAm4JGq+vuuob3A8SeCtgC3d9Uvb08VrQeeb5eT9gEbkixvN443tJokaYEs7WPt24A/BR5M8p1W+yRwHXBrkq3Ak8D729gdwDuBMeAF4AqAqppMcg1wb5t3dVVN9tGXJGmOeg6Dqvp3IDMMXzTN/AK2z3CsXcCuXnuRJPXHbyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSeIkCoMkm5I8mmQsyZWD7keSFpOTIgySLAE+C1wMrAMuS7JusF1J0uJxUoQBcAEwVlWPVdXPgD3A5gH3JEmLxtJBN9CsBA537Y8DF06dlGQbsK3t/k+SRxegt8XgTOCHg25iNvn0oDvQgPjnc3797nTFkyUMTkhV7QR2DrqPU02S0aoaGXQf0nT887kwTpbLREeA1V37q1pNkrQATpYwuBdYm2RNktOAS4G9A+5JkhaNk+IyUVUdS/JhYB+wBNhVVQcH3NZi4qU3ncz887kAUlWD7kGSNGAny2UiSdIAGQaSJMNAknSS3EDWwkryZjrf8F7ZSkeAvVX1yOC6kjRIfjJYZJJ8gs6v+whwT3sFuMVfEKiTWZIrBt3DqcyniRaZJN8Hzq2qn0+pnwYcrKq1g+lMemlJnqqqswfdx6nKy0SLzy+ANwJPTqmf1cakgUnywExDwIqF7GWxMQwWn48CdyY5xC9/OeDZwDnAhwfWldSxAtgIHJ1SD/AfC9/O4mEYLDJV9fUkv0/n14Z330C+t6peHFxnEgBfAV5bVd+ZOpDkmwvfzuLhPQNJkk8TSZIMA0kShoEkCcNAkoRhIEkC/g8jO6AFOzoXxgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "02utcblwtA-C"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "train_df, val_df = train_test_split(df, test_size=0.2)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lApGygTzntUY"
      },
      "source": [
        "get_samples = lambda df : (df[\"headline\"].values, df[\"is_sarcastic\"].values)\n",
        "train_data, train_labels = get_samples(train_df)\n",
        "val_data, val_labels = get_samples(val_df)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gjERKAMUo8AB"
      },
      "source": [
        "## Data Tokenization & Batchifying"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vg4xkMvgo_S2"
      },
      "source": [
        "### Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "byYH4g0vnTKj"
      },
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ouv7DK0ngf4"
      },
      "source": [
        "VOCAB_SIZE = 10000\n",
        "tokenizer = Tokenizer(VOCAB_SIZE, oov_token=\"<OOV>\")\n",
        "tokenizer.fit_on_texts(train_data)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QI6zH401pCV7"
      },
      "source": [
        "### Create samples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9KlQZCFzoSCc",
        "outputId": "544ad160-46e2-455d-9305-220510fe12aa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "pd.Series(train_data).apply(lambda x : len(x)).describe()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    21367.000000\n",
              "mean        60.897927\n",
              "std         19.275282\n",
              "min          7.000000\n",
              "25%         48.000000\n",
              "50%         61.000000\n",
              "75%         73.000000\n",
              "max        254.000000\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "05AAweA1n0jj"
      },
      "source": [
        "MAX_SEQ_LEN = 128\n",
        "BUFFER_SIZE = 1024\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "def create_tfds(tokenizer, X, y, padding=False):\n",
        "  if padding:\n",
        "    X = pad_sequences(tokenizer.texts_to_sequences(X), maxlen=MAX_SEQ_LEN, padding=\"post\")\n",
        "  return tf.data.Dataset.from_tensor_slices((X, y)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
        "\n",
        "ds_train = create_tfds(tokenizer, train_data, train_labels, padding=True)\n",
        "ds_val = create_tfds(tokenizer, val_data, val_labels, padding=True)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LFD4DteZpHNW"
      },
      "source": [
        "## Model Architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xkSiZ9jPpV6S",
        "outputId": "6fec9cc6-2774-4b11-f26b-fee6ab6a41f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Embedding, Dropout, GlobalAveragePooling1D, Dense\n",
        "\n",
        "EMBED_DIM = 64\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=VOCAB_SIZE, output_dim=EMBED_DIM))\n",
        "model.add(GlobalAveragePooling1D())\n",
        "model.add(Dense(64, activation=\"swish\"))\n",
        "model.add(Dropout(0.3))\n",
        "model.add(Dense(1, activation=\"sigmoid\"))\n",
        "model.summary()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, None, 64)          640000    \n",
            "_________________________________________________________________\n",
            "global_average_pooling1d (Gl (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 64)                4160      \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 1)                 65        \n",
            "=================================================================\n",
            "Total params: 644,225\n",
            "Trainable params: 644,225\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KmmkS5-RpyE9"
      },
      "source": [
        "model.compile(optimizer=\"rmsprop\", loss=\"binary_crossentropy\", metrics=[\"acc\"])"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9PqJhlA6qVUK",
        "outputId": "2ec2f113-e1d7-408d-840c-8cdf74433ef2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        }
      },
      "source": [
        "callbacks = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=7)\n",
        "model.fit(ds_train, epochs=20, validation_data=ds_val, callbacks=[callbacks]);"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.6731 - acc: 0.5682 - val_loss: 0.6311 - val_acc: 0.6221\n",
            "Epoch 2/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.5157 - acc: 0.7784 - val_loss: 0.4368 - val_acc: 0.7980\n",
            "Epoch 3/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.3800 - acc: 0.8352 - val_loss: 0.3666 - val_acc: 0.8424\n",
            "Epoch 4/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.3310 - acc: 0.8607 - val_loss: 0.3491 - val_acc: 0.8512\n",
            "Epoch 5/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.3027 - acc: 0.8740 - val_loss: 0.3347 - val_acc: 0.8575\n",
            "Epoch 6/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2840 - acc: 0.8815 - val_loss: 0.3452 - val_acc: 0.8508\n",
            "Epoch 7/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2688 - acc: 0.8905 - val_loss: 0.3666 - val_acc: 0.8401\n",
            "Epoch 8/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2581 - acc: 0.8937 - val_loss: 0.3400 - val_acc: 0.8583\n",
            "Epoch 9/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2486 - acc: 0.8986 - val_loss: 0.3400 - val_acc: 0.8575\n",
            "Epoch 10/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2408 - acc: 0.9036 - val_loss: 0.3845 - val_acc: 0.8407\n",
            "Epoch 11/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2365 - acc: 0.9041 - val_loss: 0.3356 - val_acc: 0.8641\n",
            "Epoch 12/20\n",
            "334/334 [==============================] - 2s 7ms/step - loss: 0.2295 - acc: 0.9070 - val_loss: 0.3353 - val_acc: 0.8652\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "durturicqZhL",
        "outputId": "d9486d4e-3175-46ad-eec9-406aa88e1aed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "model.evaluate(ds_val)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "84/84 [==============================] - 0s 2ms/step - loss: 0.3353 - acc: 0.8652\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3352658450603485, 0.865218997001648]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2UNcclk4vplT"
      },
      "source": [
        "### RNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kqkcCp0svvYj"
      },
      "source": [
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, GRU, Dropout, Dense, SpatialDropout1D\n",
        "\n",
        "EMBED_DIM = 64\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=VOCAB_SIZE, output_dim=EMBED_DIM))\n",
        "model.add(Bidirectional(GRU(64, return_sequences=True)))\n",
        "model.add(Bidirectional(GRU(64)))\n",
        "model.add(Dense(32, activation=\"swish\"))\n",
        "model.add(Dropout(0.4))\n",
        "model.add(Dense(1, activation=\"sigmoid\"))"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hZyCM1JFvvYz",
        "outputId": "4731789f-d23b-4f7f-ee17-b7bece361903",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "model.compile(optimizer=\"rmsprop\", loss=\"binary_crossentropy\", metrics=[\"acc\"])\n",
        "model.summary()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, None, 64)          640000    \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, None, 128)         49920     \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 128)               74496     \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 32)                4128      \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 32)                0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1)                 33        \n",
            "=================================================================\n",
            "Total params: 768,577\n",
            "Trainable params: 768,577\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u9r5o49lvvY3",
        "outputId": "20c0c7e5-a806-470f-9ca8-a7f974a9ae70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "callbacks = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=4)\n",
        "model.fit(ds_train, epochs=20, validation_data=ds_val, callbacks=[callbacks]);"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "334/334 [==============================] - 12s 35ms/step - loss: 0.4014 - acc: 0.8117 - val_loss: 0.3129 - val_acc: 0.8656\n",
            "Epoch 2/20\n",
            "334/334 [==============================] - 10s 31ms/step - loss: 0.2520 - acc: 0.8969 - val_loss: 0.3036 - val_acc: 0.8701\n",
            "Epoch 3/20\n",
            "334/334 [==============================] - 10s 31ms/step - loss: 0.2044 - acc: 0.9195 - val_loss: 0.3073 - val_acc: 0.8738\n",
            "Epoch 4/20\n",
            "334/334 [==============================] - 10s 31ms/step - loss: 0.1728 - acc: 0.9333 - val_loss: 0.3357 - val_acc: 0.8688\n",
            "Epoch 5/20\n",
            "334/334 [==============================] - 10s 31ms/step - loss: 0.1478 - acc: 0.9454 - val_loss: 0.3608 - val_acc: 0.8609\n",
            "Epoch 6/20\n",
            "334/334 [==============================] - 10s 30ms/step - loss: 0.1245 - acc: 0.9553 - val_loss: 0.4131 - val_acc: 0.8579\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ec7YMnzq-RMz",
        "outputId": "4c60b0d7-29df-48d0-8300-3075ee704e69",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "model.evaluate(ds_val)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "84/84 [==============================] - 1s 12ms/step - loss: 0.4131 - acc: 0.8579\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.4130658209323883, 0.857918381690979]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o_ZHnYI8ncpl"
      },
      "source": [
        "### Transfer learning w. pretrained embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "atVA7Hx6AFrd"
      },
      "source": [
        "ds_train = create_tfds(tokenizer, train_data, train_labels, padding=False)\n",
        "ds_val = create_tfds(tokenizer, val_data, val_labels, padding=False)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Smp7hTc4SR9R",
        "outputId": "ef116f5f-0aab-427b-a0f0-45b93020598c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "import tensorflow_hub as hub\n",
        "embedding_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\"\n",
        "\n",
        "model = Sequential()\n",
        "model.add(hub.KerasLayer(embedding_url, input_shape=(), dtype=tf.string, trainable=True))\n",
        "model.add(Dense(64, activation=\"swish\"))\n",
        "model.add(Dropout(0.3))\n",
        "model.add(Dense(1, activation=\"sigmoid\"))\n",
        "model.summary()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "keras_layer (KerasLayer)     (None, 512)               256797824 \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 64)                32832     \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 1)                 65        \n",
            "=================================================================\n",
            "Total params: 256,830,721\n",
            "Trainable params: 256,830,721\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EpK3E9TPrBct"
      },
      "source": [
        "model.compile(optimizer=\"rmsprop\", loss=\"binary_crossentropy\", metrics=[\"acc\"])"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dP2OvjUeylmZ",
        "outputId": "20e8afc3-d4f3-4160-e5e7-366c74ff2975",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "callbacks = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=3)\n",
        "model.fit(ds_train, epochs=20, validation_data=ds_val, callbacks=[callbacks]);"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "334/334 [==============================] - 24s 72ms/step - loss: 0.3984 - acc: 0.8191 - val_loss: 0.3126 - val_acc: 0.8647\n",
            "Epoch 2/20\n",
            "334/334 [==============================] - 24s 71ms/step - loss: 0.2204 - acc: 0.9103 - val_loss: 0.2756 - val_acc: 0.8854\n",
            "Epoch 3/20\n",
            "334/334 [==============================] - 24s 71ms/step - loss: 0.1341 - acc: 0.9493 - val_loss: 0.2767 - val_acc: 0.8897\n",
            "Epoch 4/20\n",
            "334/334 [==============================] - 24s 71ms/step - loss: 0.0790 - acc: 0.9715 - val_loss: 0.2899 - val_acc: 0.8875\n",
            "Epoch 5/20\n",
            "334/334 [==============================] - 24s 71ms/step - loss: 0.0449 - acc: 0.9844 - val_loss: 0.3571 - val_acc: 0.8830\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6WrGVYpFeWWH",
        "outputId": "f7b4b1eb-0712-450b-b7b5-246d507a429c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "model.evaluate(ds_val)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "84/84 [==============================] - 1s 9ms/step - loss: 0.3571 - acc: 0.8830\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3571247458457947, 0.8830026388168335]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    }
  ]
}