Sentiment Cloud.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "DDWcMWR1Akkd",
        "colab_type": "code",
        "outputId": "2df7d228-812e-4ddc-f646-28f1351108f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "4WstZFlgD-mh",
        "colab_type": "code",
        "outputId": "a0c11f5c-ed77-4e39-91d2-06a736907e47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[0m\u001b[01;34mdrive\u001b[0m/  \u001b[01;34msample_data\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "HzzPUF69EDWI",
        "colab_type": "code",
        "outputId": "420c9fc6-7097-42a0-c81f-2ba3cfd7bbe3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install torch torchvision"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting torch\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/60/66415660aa46b23b5e1b72bc762e816736ce8d7260213e22365af51e8f9c/torch-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (591.8MB)\n",
            "\u001b[K    100% |████████████████████████████████| 591.8MB 24kB/s \n",
            "tcmalloc: large alloc 1073750016 bytes == 0x60a0c000 @  0x7f4405a6e2a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n",
            "\u001b[?25hCollecting torchvision\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ca/0d/f00b2885711e08bd71242ebe7b96561e6f6d01fdb4b9dcf4d37e2e13c5e1/torchvision-0.2.1-py2.py3-none-any.whl (54kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 22.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.14.6)\n",
            "Collecting pillow>=4.1.1 (from torchvision)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/62/94/5430ebaa83f91cc7a9f687ff5238e26164a779cca2ef9903232268b0a318/Pillow-5.3.0-cp36-cp36m-manylinux1_x86_64.whl (2.0MB)\n",
            "\u001b[K    100% |████████████████████████████████| 2.0MB 4.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.11.0)\n",
            "Installing collected packages: torch, pillow, torchvision\n",
            "  Found existing installation: Pillow 4.0.0\n",
            "    Uninstalling Pillow-4.0.0:\n",
            "      Successfully uninstalled Pillow-4.0.0\n",
            "Successfully installed pillow-5.3.0 torch-1.0.0 torchvision-0.2.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "3ciW63qIETUr",
        "colab_type": "code",
        "outputId": "a92d5d7d-0f41-4745-8ad0-dc3c7f85b528",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 183
        }
      },
      "cell_type": "code",
      "source": [
        "#Doing this because, sometimes we get an error 'module 'PIL.Image' has no attribute 'register_extensions'\n",
        "#uninstall the old one\n",
        "!pip uninstall -y Pillow\n",
        "# install the new one\n",
        "!pip install Pillow==4.1.1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling Pillow-5.3.0:\n",
            "  Successfully uninstalled Pillow-5.3.0\n",
            "Collecting Pillow==4.1.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/36/e5/88b3d60924a3f8476fa74ec086f5fbaba56dd6cee0d82845f883b6b6dd18/Pillow-4.1.1-cp36-cp36m-manylinux1_x86_64.whl (5.7MB)\n",
            "\u001b[K    100% |████████████████████████████████| 5.7MB 5.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow==4.1.1) (0.46)\n",
            "Installing collected packages: Pillow\n",
            "Successfully installed Pillow-4.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Qx_MCDNQAaE9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader, TensorDataset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I_IXky6QAaFG",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Visualising the data"
      ]
    },
    {
      "metadata": {
        "id": "LWTcqYxOAaFI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "with open('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Sentiment Analysis Movie/reviews.txt') as f:\n",
        "    reviews = f.read()\n",
        "\n",
        "with open('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Sentiment Analysis Movie/labels.txt') as f:\n",
        "    labels = f.read()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Wc699bziAaFN",
        "colab_type": "code",
        "outputId": "39d98ff0-f69f-48ba-b7c5-3672a99d2541",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        }
      },
      "cell_type": "code",
      "source": [
        "print(reviews[:2000])\n",
        "print()\n",
        "print(labels[:20])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "bromwell high is a cartoon comedy . it ran at the same time as some other programs about school life  such as  teachers  . my   years in the teaching profession lead me to believe that bromwell high  s satire is much closer to reality than is  teachers  . the scramble to survive financially  the insightful students who can see right through their pathetic teachers  pomp  the pettiness of the whole situation  all remind me of the schools i knew and their students . when i saw the episode in which a student repeatedly tried to burn down the school  i immediately recalled . . . . . . . . . at . . . . . . . . . . high . a classic line inspector i  m here to sack one of your teachers . student welcome to bromwell high . i expect that many adults of my age think that bromwell high is far fetched . what a pity that it isn  t   \n",
            "story of a man who has unnatural feelings for a pig . starts out with a opening scene that is a terrific example of absurd comedy . a formal orchestra audience is turned into an insane  violent mob by the crazy chantings of it  s singers . unfortunately it stays absurd the whole time with no general narrative eventually making it just too off putting . even those from the era should be turned off . the cryptic dialogue would make shakespeare seem easy to a third grader . on a technical level it  s better than you might think with some good cinematography by future great vilmos zsigmond . future stars sally kirkland and frederic forrest can be seen briefly .  \n",
            "homelessness  or houselessness as george carlin stated  has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school  work  or vote for the matter . most people think of the homeless as just a lost cause while worrying about things such as racism  the war on iraq  pressuring kids to succeed  technology  the elections  inflation  or worrying if they  ll be next to end up on the streets .  br    br   but what if y\n",
            "\n",
            "positive\n",
            "negative\n",
            "po\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9S3y2g7GAaFX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We see that the reviews are saperated by new lines so are labels"
      ]
    },
    {
      "metadata": {
        "id": "mEedeRzGAaFZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Data Preprocessing\n",
        "We need data to be in proper format to feed into the neural network. So encode the words into a unique number and we will also clean a bit\n",
        "1. First we remove the punctuations\n",
        "2. Next we remove new line characters and combine everything into one\n",
        "3. Next we will create a list of words"
      ]
    },
    {
      "metadata": {
        "id": "wnYT9Ju0AaFb",
        "colab_type": "code",
        "outputId": "edd639af-e4c2-4f25-af5f-801e5ff00c73",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "#removing punctuations\n",
        "from string import punctuation\n",
        "print('Punctuations: ', punctuation)\n",
        "\n",
        "reviews = reviews.lower()\n",
        "all_text = ''.join([ch for ch in reviews if ch not in punctuation])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Punctuations:  !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "PQGS5zajAaFi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#joining by new lines, combining and splitting by words\n",
        "reviews_split = all_text.split('\\n')\n",
        "all_text = ' '.join(reviews_split)\n",
        "all_text = all_text.split()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UyEfQ3ANAaFn",
        "colab_type": "code",
        "outputId": "c2dfc8ad-5af4-4c95-969b-27305393487d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "all_text[:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['bromwell', 'high', 'is', 'a', 'cartoon', 'comedy', 'it', 'ran', 'at', 'the']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "metadata": {
        "id": "Pb1LVFIhAaFu",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Encode the data\n",
        "Here we convert each unique word into an integer so that we can feed into the neural network. We use `Counter` so that the most frequent word will be given the lowest integer."
      ]
    },
    {
      "metadata": {
        "id": "yibW22fPAaFv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "\n",
        "vocab2int = {}\n",
        "i = 1\n",
        "for x, y in Counter(all_text).most_common():\n",
        "    vocab2int[x] = i\n",
        "    i+=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zOzEY8-GAaF1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "reviews_ints = []\n",
        "for sent in reviews_split:\n",
        "    a = []\n",
        "    for word in sent.split():\n",
        "        a.append(vocab2int[word])\n",
        "        \n",
        "    reviews_ints.append(a)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bH_sd09nAaF5",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Test code for above implementation"
      ]
    },
    {
      "metadata": {
        "id": "UO9gmT8MAaF7",
        "colab_type": "code",
        "outputId": "0d4ecddb-031d-44f2-c192-2832d797a816",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "cell_type": "code",
      "source": [
        "print('Unique words: ', len(vocab2int))\n",
        "print()\n",
        "\n",
        "print('Tokenized view:\\n', reviews_ints[:1])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique words:  74072\n",
            "\n",
            "Tokenized view:\n",
            " [[21025, 308, 6, 3, 1050, 207, 8, 2138, 32, 1, 171, 57, 15, 49, 81, 5785, 44, 382, 110, 140, 15, 5194, 60, 154, 9, 1, 4975, 5852, 475, 71, 5, 260, 12, 21025, 308, 13, 1978, 6, 74, 2395, 5, 613, 73, 6, 5194, 1, 24103, 5, 1983, 10166, 1, 5786, 1499, 36, 51, 66, 204, 145, 67, 1199, 5194, 19869, 1, 37442, 4, 1, 221, 883, 31, 2988, 71, 4, 1, 5787, 10, 686, 2, 67, 1499, 54, 10, 216, 1, 383, 9, 62, 3, 1406, 3686, 783, 5, 3483, 180, 1, 382, 10, 1212, 13583, 32, 308, 3, 349, 341, 2913, 10, 143, 127, 5, 7690, 30, 4, 129, 5194, 1406, 2326, 5, 21025, 308, 10, 528, 12, 109, 1448, 4, 60, 543, 102, 12, 21025, 308, 6, 227, 4146, 48, 3, 2211, 12, 8, 215, 23]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Om9WfzPaAaGC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Encoding the labels\n",
        "We have only two labels. Those must be encoded. We will convert positive label to 1 and negative label to 0"
      ]
    },
    {
      "metadata": {
        "id": "mkEZvVL-AaGE",
        "colab_type": "code",
        "outputId": "5dd1ba6a-8523-45cc-dcf5-e4c275b8c287",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "labels_encoded = []\n",
        "labels_split = labels.split('\\n')\n",
        "\n",
        "for x in labels_split:\n",
        "    if x == 'positive':\n",
        "        labels_encoded.append(1)\n",
        "    else:\n",
        "        labels_encoded.append(0)\n",
        "\n",
        "labels_encoded[:3]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 0, 1]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "BJAUN3hnAaGK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "You can verify these outputs with the starting outputted values"
      ]
    },
    {
      "metadata": {
        "id": "sHkHLOecAaGM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Removing the outliers\n",
        "We want to make sure that our model will not get some outliers i.e., values that are extremely big or too small.\n",
        "\n",
        "For other values, we want to make sure that all the reviews should be of same lenth."
      ]
    },
    {
      "metadata": {
        "id": "oJeonsRGAaGN",
        "colab_type": "code",
        "outputId": "36fc61f5-b286-4011-98b2-54c71658c329",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "review_lengths = Counter([len(x) for x in reviews_ints])\n",
        "print('Reviews with zero length: ', review_lengths[0])\n",
        "print('Reviews with max length: ', max(review_lengths))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reviews with zero length:  1\n",
            "Reviews with max length:  2514\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Ed2-XDD5AaGV",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "As we see that we have one review that has zero length and also we see that the maximum length is very large. So we want to remove the reviews with shorter length and truncate the reviews with longer length so that our model will learn more effectively"
      ]
    },
    {
      "metadata": {
        "id": "Eg-uug4PAaGZ",
        "colab_type": "code",
        "outputId": "59c4bcc0-7759-455c-d82b-dbf95e8c63bd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "print('Number of reviews before removing the outliers: ', len(reviews_ints))\n",
        "\n",
        "#removing the reviews with zero length\n",
        "non_zero_idx = [ii for ii, rev in enumerate(reviews_ints) if len(rev)!=0]\n",
        "\n",
        "reviews_ints = [reviews_ints[ii] for ii in non_zero_idx]\n",
        "labels_encoded = np.array([labels_encoded[ii] for ii in non_zero_idx])\n",
        "\n",
        "print('Number of reviews after removing the outliers: ', len(reviews_ints))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of reviews before removing the outliers:  25001\n",
            "Number of reviews after removing the outliers:  25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "pDhC0xa7AaGi",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Padding and truncating reviews\n",
        "Now we want to truncate the reviews that are too long to some specified length and the reviews that are shorter than specified length then we will pad them with 0s. The good sequence length will be 20"
      ]
    },
    {
      "metadata": {
        "id": "z8jaUKdcAaGj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pad_features(reviews_ints, seq_length):\n",
        "    '''\n",
        "    This function will return an array of reviews that are either truncated to seq_length\n",
        "    or padded with 0s\n",
        "    '''\n",
        "    features = []\n",
        "    \n",
        "    for rev in reviews_ints:\n",
        "        if len(rev) >= seq_length:\n",
        "            features.append(rev[:seq_length])\n",
        "        else:\n",
        "            features.append([0]*(seq_length-len(rev)) + rev)\n",
        "    \n",
        "    return np.array(features)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XR62GX5aAaGq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Test code to test the above function"
      ]
    },
    {
      "metadata": {
        "id": "0nDUk5psAaGs",
        "colab_type": "code",
        "outputId": "31a8f120-17b9-4510-ce05-4d8555a2dcc9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "seq_length = 200\n",
        "\n",
        "features = pad_features(reviews_ints, seq_length)\n",
        "\n",
        "#features should have same rows as reviews_ints\n",
        "print('Rows in features: {} -- Rows in reviews_ints: {}'.format(len(features), len(reviews_ints)))\n",
        "\n",
        "#features should have same number of columns as sequence length\n",
        "print('Columns in features: {} -- Sequence length: {}'.format(len(features[0]), seq_length))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rows in features: 25000 -- Rows in reviews_ints: 25000\n",
            "Columns in features: 200 -- Sequence length: 200\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "h5M7OiUdAaG2",
        "colab_type": "code",
        "outputId": "43f4fda6-e215-4392-b7dc-1ff379bdb4ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 563
        }
      },
      "cell_type": "code",
      "source": [
        "print(features[:30, :10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [22382    42 46418    15   706 17139  3389    47    77    35]\n",
            " [ 4505   505    15     3  3342   162  8312  1652     6  4819]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   54    10    14   116    60   798   552    71   364     5]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    1   330   578    34     3   162   748  2731     9   325]\n",
            " [    9    11 10171  5305  1946   689   444    22   280   673]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    1   307 10399  2069  1565  6202  6528  3288 17946 10628]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   21   122  2069  1565   515  8181    88     6  1325  1182]\n",
            " [    1    20     6    76    40     6    58    81    95     5]\n",
            " [   54    10    84   329 26230 46427    63    10    14   614]\n",
            " [   11    20     6    30  1436 32317  3769   690 15100     6]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   40    26   109 17952  1422     9     1   327     4   125]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   10   499     1   307 10399    55    74     8    13    30]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QlMUSwXcAaG9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Splitting into training, validation and test sets\n",
        "We will split the data into three parts. We will declare a `split_frac` that will tell how much of the data should be in training data and the remaining data is split into two halves, one for validation and other for testing\n",
        "\n",
        "First I have generated random indexes of size 80% of length of features. Then I have used these indexes to extract traaining samples and out of remaining samples, I have taken 50% to validation and other 50% test set."
      ]
    },
    {
      "metadata": {
        "id": "Olhd-ZwNAaG-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "split_frac = 0.8\n",
        "\n",
        "train_idx = np.random.choice(np.arange(len(features)), int(len(features)*split_frac), replace=False)\n",
        "\n",
        "train_x = np.array([features[ii] for ii in train_idx])\n",
        "train_y = np.array([labels_encoded[ii] for ii in train_idx])\n",
        "\n",
        "valid_reviews = np.array([features[ii] for ii in range(len(features)) if ii not in train_idx])\n",
        "valid_labels  = np.array([labels_encoded[ii] for ii in range(len(features)) if ii not in train_idx])\n",
        "\n",
        "valid_x = valid_reviews[:int(valid_reviews.shape[0]/2)]\n",
        "test_x = valid_reviews[int(valid_reviews.shape[0]/2):]\n",
        "\n",
        "valid_y = valid_labels[:int(valid_reviews.shape[0]/2)]\n",
        "test_y = valid_labels[int(valid_reviews.shape[0]/2):]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7tYMja80AaHD",
        "colab_type": "code",
        "outputId": "9937919d-7e65-46a9-83ec-fa079a0a4af7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "cell_type": "code",
      "source": [
        "#printing the shape of splitted data\n",
        "print('Feature Shapes\\n',\n",
        "      'Train set:', train_x.shape,\n",
        "      '\\nValid set:', valid_x.shape,\n",
        "      '\\nTest set:', test_x.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Feature Shapes\n",
            " Train set: (20000, 200) \n",
            "Valid set: (2500, 200) \n",
            "Test set: (2500, 200)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ezDLdJsHAaHI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## DataLoaders and Batching\n",
        "Now we have x,y that is inputs and targets, we convert them to dataset `TensorDataset` class, next we can use this dataset to pass to `DataLoader` class and get batches of data"
      ]
    },
    {
      "metadata": {
        "id": "uFdlda0aAaHL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))\n",
        "valid_data = TensorDataset(torch.from_numpy(valid_x), torch.from_numpy(valid_y))\n",
        "test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))\n",
        "\n",
        "#defining the batch size\n",
        "batch_size = 50\n",
        "\n",
        "trainloader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
        "validloader = DataLoader(valid_data, batch_size=batch_size, shuffle=True)\n",
        "testloader = DataLoader(test_data, batch_size=batch_size, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ub58GZBOAaHR",
        "colab_type": "code",
        "outputId": "26481123-4328-44f6-fada-ae113e997058",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 290
        }
      },
      "cell_type": "code",
      "source": [
        "#obtaining one batch of training data\n",
        "sample_x, sample_y = next(iter(trainloader))\n",
        "\n",
        "print('Shape of sample_x: ', sample_x.shape)\n",
        "print('sample_x: \\n', sample_x)\n",
        "print()\n",
        "print('Shape of sample_y: ', sample_y.shape)\n",
        "print('sample_y: \\n', sample_y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of sample_x:  torch.Size([50, 200])\n",
            "sample_x: \n",
            " tensor([[   0,    0,    0,  ...,  348,   98, 2404],\n",
            "        [   3, 3154,  382,  ...,   34,    3, 7939],\n",
            "        [   0,    0,    0,  ...,   55,  698,  493],\n",
            "        ...,\n",
            "        [  60,  191,    3,  ...,   77,   37,   74],\n",
            "        [   8,   13,  428,  ...,  222,   39,    5],\n",
            "        [   0,    0,    0,  ...,    3,   45,    4]])\n",
            "\n",
            "Shape of sample_y:  torch.Size([50])\n",
            "sample_y: \n",
            " tensor([0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0,\n",
            "        1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0,\n",
            "        0, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "mXZYhV60AaHY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Defining the sentiment network\n",
        "We have to define the network. Here we will use the `embedding layer` as input to the `lstm` because we have a lot of words and using one-hot encoding to these would be inefficient.\n",
        "1. We defined the embedding layer to convert the words tokens(integers) to embeddings of some specific size.\n",
        "2. Defined a lstm layers that takes inputs from the embedding layer\n",
        "3. Define a fully connected layer that maps lstm outputs to our desired output size\n",
        "4. Use sigmoid activation function, which turns outputs in the value of range 0-1"
      ]
    },
    {
      "metadata": {
        "id": "nXxd8cJxAaHZ",
        "colab_type": "code",
        "outputId": "d2629f86-4299-4b40-841c-700a0c1a5dc6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "#checking availability of gpu\n",
        "cuda_available = torch.cuda.is_available()\n",
        "if cuda_available:\n",
        "    print('Training on GPU')\n",
        "else:\n",
        "    print('Training on CPU')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on GPU\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7xoqVp1NAaHi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class SentimentRNN(nn.Module):\n",
        "    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_p=0.3):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.output_size = output_size\n",
        "        self.n_layers = n_layers\n",
        "        self.hidden_dim = hidden_dim\n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, batch_first=True, dropout=drop_p)\n",
        "        self.fc = nn.Linear(hidden_dim, output_size)\n",
        "        \n",
        "        self.dropout = nn.Dropout(p=drop_p)\n",
        "        \n",
        "    \n",
        "    def forward(self, x, hidden):\n",
        "        \n",
        "        batch_size = x.shape[0]\n",
        "        \n",
        "        x = self.embedding(x)\n",
        "        output, hidden = self.lstm(x,hidden)\n",
        "        \n",
        "        #stack up lstm outputs\n",
        "        output = output.contiguous().view(-1, self.hidden_dim)\n",
        "        \n",
        "        output = self.dropout(output)\n",
        "        \n",
        "        \n",
        "        output = self.fc(output)\n",
        "        output = F.sigmoid(output)\n",
        "        \n",
        "        #reshape to be batch first\n",
        "        output = output.view(batch_size, -1)\n",
        "        output = output[:, -1] #get the last batch\n",
        "        \n",
        "        \n",
        "        return output, hidden\n",
        "    \n",
        "    def init_hidden(self, batch_size):\n",
        "        weight = next(self.parameters()).data\n",
        "        \n",
        "        if cuda_available:\n",
        "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
        "                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
        "        else:\n",
        "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),\n",
        "                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())\n",
        "        return hidden"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "42YlPXfuAaHl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Instantiate the model\n",
        "Here, we'll instantiate the network. First up, defining the hyperparameters.\n",
        "1. vocab_size: Size of our vocabulary or the range of values for our input, word tokens\n",
        "2. output_size: Size of our desired output; the number of class scores we want to output (pos/neg)\n",
        "3. embedding_dim: Number of columns in the embedding lookup table; size of our embeddings.\n",
        "4. hidden_dim: Number of units in the hidden layers of our LSTM cells. Usually larger is better performance wise. Common values are 128, 256, 512, etc.\n",
        "5. n_layers: Number of LSTM layers in the network. Typically between 1-3"
      ]
    },
    {
      "metadata": {
        "id": "pqnpYV0eAaHn",
        "colab_type": "code",
        "outputId": "b7347969-0d95-42d1-a712-fc3d3bee09eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "#instantiate the model with hyper parameters\n",
        "vocab_size = len(vocab2int)+1 #+1 for the padding 0\n",
        "output_size = 1\n",
        "embedding_dim = 400\n",
        "hidden_dim = 256\n",
        "n_layers = 2\n",
        "\n",
        "model = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)\n",
        "print(model)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SentimentRNN(\n",
            "  (embedding): Embedding(74073, 400)\n",
            "  (lstm): LSTM(400, 256, num_layers=2, batch_first=True, dropout=0.3)\n",
            "  (fc): Linear(in_features=256, out_features=1, bias=True)\n",
            "  (dropout): Dropout(p=0.3)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "yR6WhUDaAaHs",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Training\n",
        "Now it's time to train our model. \n",
        "This time we use different loss function for only one output called `Binary Cross Entropy Loss`.\n",
        "\n",
        "The hyper parameters for training are:\n",
        "1. lr: learning rate\n",
        "2. epochs: the number of epochs\n",
        "3. clip: the maximum gradient value to clip at"
      ]
    },
    {
      "metadata": {
        "id": "S5XCMRPdAaHt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train(model,trainloader, validloader, batch_size,clip=5, print_every=200, epochs=4, lr=0.001):\n",
        "    \n",
        "    if cuda_available:\n",
        "        model = model.cuda()\n",
        "    \n",
        "    criterion = nn.BCELoss()\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "    \n",
        "    counter = 0 \n",
        "    for e in range(epochs):\n",
        "        \n",
        "        hidden = model.init_hidden(batch_size)\n",
        "        for x, y in trainloader:\n",
        "            model.train()\n",
        "            counter += 1\n",
        "            \n",
        "            if cuda_available:\n",
        "                x, y = x.cuda(), y.cuda()\n",
        "            \n",
        "            hidden = tuple([each.data for each in hidden])\n",
        "            output, hidden = model(x, hidden)\n",
        "            loss = criterion(output.squeeze(), y.float())\n",
        "            optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
        "            optimizer.step()\n",
        "            \n",
        "            if counter%print_every==0:\n",
        "                \n",
        "                val_h = model.init_hidden(batch_size)\n",
        "                val_losses = []\n",
        "                num_correct = 0\n",
        "                model.eval()\n",
        "                \n",
        "                for x, y in validloader:\n",
        "                    \n",
        "                    val_h = tuple([each.data for each in val_h])\n",
        "                    \n",
        "                    if cuda_available:\n",
        "                        x, y = x.cuda(), y.cuda()\n",
        "                        \n",
        "                    output, hidden = model(x, val_h)\n",
        "                    loss_valid = criterion(output.squeeze(), y.float())\n",
        "                    val_losses.append(loss_valid.item())\n",
        "                    \n",
        "                    #convert output to probabilities 0 or 1\n",
        "                    pred = torch.round(output.squeeze())\n",
        "                    correct = pred.eq(y.float().view_as(pred))\n",
        "                    correct = np.squeeze(correct.cpu().numpy() if cuda_available else correct.numpy())\n",
        "                    num_correct += np.sum(correct)\n",
        "                    \n",
        "                print('Epoch: {}/{}'.format(e+1, epochs),\n",
        "                        'Training Loss: {}'.format(loss.item()),\n",
        "                        'Validation loss: {}'.format(np.mean(val_losses)),\n",
        "                        'Validation Acc: {:.3f}'.format(num_correct/len(validloader.dataset)))\n",
        "                \n",
        "                \n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cfqvTUaUAaHy",
        "colab_type": "code",
        "outputId": "b6863c06-389f-47aa-f3dd-010939273af6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        }
      },
      "cell_type": "code",
      "source": [
        "train(model, trainloader, validloader, batch_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1332: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
            "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1/4 Training Loss: 0.7052900195121765 Validation loss: 0.6954298067092896 Validation Acc: 0.497\n",
            "Epoch: 1/4 Training Loss: 0.6976535320281982 Validation loss: 0.695543133020401 Validation Acc: 0.498\n",
            "Epoch: 2/4 Training Loss: 0.6718894839286804 Validation loss: 0.6830545449256897 Validation Acc: 0.615\n",
            "Epoch: 2/4 Training Loss: 0.44698867201805115 Validation loss: 0.4678158468008041 Validation Acc: 0.775\n",
            "Epoch: 3/4 Training Loss: 0.3892812430858612 Validation loss: 0.43840566635131833 Validation Acc: 0.815\n",
            "Epoch: 3/4 Training Loss: 0.3334687352180481 Validation loss: 0.3969148832559586 Validation Acc: 0.826\n",
            "Epoch: 4/4 Training Loss: 0.21826237440109253 Validation loss: 0.4413024526834488 Validation Acc: 0.816\n",
            "Epoch: 4/4 Training Loss: 0.18218857049942017 Validation loss: 0.45038269996643066 Validation Acc: 0.837\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "j5t6WH9KZJMq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Testing**"
      ]
    },
    {
      "metadata": {
        "id": "QnPpDPetOI3x",
        "colab_type": "code",
        "outputId": "3f010f7a-956e-4ccb-feb7-2ec204b9bef9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "cell_type": "code",
      "source": [
        "test_losses = []\n",
        "num_correct = 0\n",
        "criterion = nn.BCELoss()\n",
        "\n",
        "test_h = model.init_hidden(batch_size)\n",
        "for x, y in testloader:\n",
        "    \n",
        "    if cuda_available:\n",
        "      x, y = x.cuda(), y.cuda()\n",
        "      \n",
        "    test_h = tuple([each.data for each in test_h])\n",
        "    \n",
        "    output, test_h = model(x, test_h)\n",
        "    test_loss = criterion(output.squeeze(), y.float())\n",
        "    \n",
        "    test_losses.append(test_loss.item())\n",
        "    \n",
        "    pred = torch.round(output)\n",
        "    correct = pred.eq(y.float().view_as(pred))\n",
        "    correct = np.squeeze(correct.cpu().numpy() if cuda_available else correct.numpy())\n",
        "    \n",
        "    num_correct += np.sum(correct)\n",
        "    \n",
        "    \n",
        "print('Test Loss: {}'.format(np.mean(test_losses)))\n",
        "print('Test Acc: {:.3f}'.format(num_correct/len(testloader.dataset)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1332: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
            "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test Loss: 0.41093942999839783\n",
            "Test Acc: 0.847\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "DE75-THNehYm",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Inference**"
      ]
    },
    {
      "metadata": {
        "id": "-eSAVlI-axV4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#review = 'The movie is quite boring. I just came to the movie because I love Shah Rukh Khan, but now I feel that money is wasted. The fighting scenes were too old and the acting is also not good'\n",
        "review = 'Although the last movie was bad this movie is a great comeback. The direction and animations are pretty good and the songs were fine.'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IeNeQYSHfIeG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pre_process(review):\n",
        "    \n",
        "    review = review.lower()\n",
        "    review = ''.join([x for x in review if x not in punctuation])\n",
        "    review = review.split()\n",
        "    \n",
        "    rev = []\n",
        "    a = []\n",
        "    for x in review:\n",
        "        a.append(vocab2int[x])\n",
        "    rev.append(a)\n",
        "    \n",
        "    rev = pad_features(rev, 200)\n",
        "    \n",
        "    return rev"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qFKsImnGhdOE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def predict(model, review, seq_length):\n",
        "    \n",
        "    \n",
        "    \n",
        "    rev = pre_process(review)\n",
        "    rev = torch.from_numpy(rev)\n",
        "    if cuda_available:\n",
        "        model = model.cuda()\n",
        "        rev = rev.cuda()\n",
        "    \n",
        "    p_h = model.init_hidden(1)\n",
        "    output, hidden = model(rev, p_h)\n",
        "    \n",
        "    pred = torch.round(output.squeeze())\n",
        "    \n",
        "    if pred.item() == 1:\n",
        "        print('Glad you liked the movie!!')\n",
        "    else:\n",
        "        print('We are sorry that you did not like the movie.')\n",
        "        \n",
        "    print('The review was predicted with an accuracy of {}'.format(output.item()))\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "coGyFyiIh4MZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        },
        "outputId": "dd7eac87-1319-4fd1-95db-6637e09c0113"
      },
      "cell_type": "code",
      "source": [
        "predict(model, review, 200)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Glad you liked the movie!!\n",
            "The review was predicted with an accuracy of 0.9394091367721558\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1332: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
            "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "WXOk55Tli4CN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}