LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O4zUHndy2Ti8"
      },
      "source": [
        "\n",
        "# Load in and visualize the data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        },
        "id": "d9ahBn01xLJY",
        "outputId": "c94abc19-5fe7-4d56-b246-e12ba5df8536"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "data = pd.read_csv('Tweets.csv')\n",
        "data.head()"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>airline_sentiment</th>\n",
              "      <th>airline_sentiment_confidence</th>\n",
              "      <th>negativereason</th>\n",
              "      <th>negativereason_confidence</th>\n",
              "      <th>airline</th>\n",
              "      <th>airline_sentiment_gold</th>\n",
              "      <th>name</th>\n",
              "      <th>negativereason_gold</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_coord</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>tweet_location</th>\n",
              "      <th>user_timezone</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>570306133677760513</td>\n",
              "      <td>neutral</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Virgin America</td>\n",
              "      <td>NaN</td>\n",
              "      <td>cairdin</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@VirginAmerica What @dhepburn said.</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-24 11:35:52 -0800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Eastern Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>570301130888122368</td>\n",
              "      <td>positive</td>\n",
              "      <td>0.3486</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>Virgin America</td>\n",
              "      <td>NaN</td>\n",
              "      <td>jnardino</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-24 11:15:59 -0800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Pacific Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>570301083672813571</td>\n",
              "      <td>neutral</td>\n",
              "      <td>0.6837</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Virgin America</td>\n",
              "      <td>NaN</td>\n",
              "      <td>yvonnalynn</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-24 11:15:48 -0800</td>\n",
              "      <td>Lets Play</td>\n",
              "      <td>Central Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>570301031407624196</td>\n",
              "      <td>negative</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>Bad Flight</td>\n",
              "      <td>0.7033</td>\n",
              "      <td>Virgin America</td>\n",
              "      <td>NaN</td>\n",
              "      <td>jnardino</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-24 11:15:36 -0800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Pacific Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>570300817074462722</td>\n",
              "      <td>negative</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>Can't Tell</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>Virgin America</td>\n",
              "      <td>NaN</td>\n",
              "      <td>jnardino</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2015-02-24 11:14:45 -0800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Pacific Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  ...               user_timezone\n",
              "0  570306133677760513  ...  Eastern Time (US & Canada)\n",
              "1  570301130888122368  ...  Pacific Time (US & Canada)\n",
              "2  570301083672813571  ...  Central Time (US & Canada)\n",
              "3  570301031407624196  ...  Pacific Time (US & Canada)\n",
              "4  570300817074462722  ...  Pacific Time (US & Canada)\n",
              "\n",
              "[5 rows x 15 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-5icvkN4xepi"
      },
      "source": [
        "reviews = np.array(data['text'])[:14000]\n",
        "labels = np.array(data['airline_sentiment'])[:14000]"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "7NCQLxPrxesJ",
        "outputId": "c5791810-d02a-48aa-dae0-9c9fe56b6d87"
      },
      "source": [
        "data['text'].loc[14639]"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'@AmericanAir we have 8 ppl so we need 2 know how many seats are on the next flight. Plz put us on standby for 4 people on the next flight?'"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "6tluUo3fxeut",
        "outputId": "69b5253c-a145-44f4-de8c-9aaec8f417b6"
      },
      "source": [
        "data['airline_sentiment'].loc[14639]"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'neutral'"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uY2Zcy2Txew6",
        "outputId": "3ffa0fb3-d451-4308-aa3b-36bce0f4bf62"
      },
      "source": [
        "from collections import Counter\n",
        "\n",
        "Counter(labels)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'negative': 8679, 'neutral': 3017, 'positive': 2304})"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mhdsYlmG2ahd"
      },
      "source": [
        "\n",
        "# Data pre-processing\n",
        "The first step when building a neural network model is getting the data into the proper form to feed into the network. Since we're using embedding layers, we'll need to encode each word with an integer. We'll also want to clean it up a bit.\n",
        "\n",
        "Here are the processing steps, we'll want to take:\n",
        "\n",
        "We'll want to get rid of periods and extraneous punctuation.\n",
        "We'll want to remove web address, twitter id, and digit.\n",
        "First, let's remove all punctuation. Then get all the text without the newlines and split it into individual words."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5KbILXLyxezZ"
      },
      "source": [
        "punctuation = '!\"#$%&\\'()*+,-./:;<=>?[\\\\]^_`{|}~'\n",
        "\n",
        "# get rid of punctuation\n",
        "all_reviews = 'separator'.join(reviews)\n",
        "all_reviews = all_reviews.lower()\n",
        "all_text = ''.join([c for c in all_reviews if c not in punctuation])\n",
        "\n",
        "# split by new lines and spaces\n",
        "reviews_split = all_text.split('separator')\n",
        "all_text = ' '.join(reviews_split)\n",
        "\n",
        "# create a list of words\n",
        "words = all_text.split()"
      ],
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SanM7Oxtxe2a"
      },
      "source": [
        "# get rid of web address, twitter id, and digit\n",
        "new_reviews = []\n",
        "for review in reviews_split:\n",
        "    review = review.split()\n",
        "    new_text = []\n",
        "    for word in review:\n",
        "        if (word[0] != '@') & ('http' not in word) & (~word.isdigit()):\n",
        "            new_text.append(word)\n",
        "    new_reviews.append(new_text)"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SK2yPqGM2hYk"
      },
      "source": [
        "\n",
        "# Encoding the words\n",
        "The embedding lookup requires that we pass in integers to our network. The easiest way to do this is to create dictionaries that map the words in the vocabulary to integers. Then, we can convert each of our reviews into integers so they can be passed into the network."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OKMjxHisxraz"
      },
      "source": [
        "## Build a dictionary that maps words to integers\n",
        "counts = Counter(words)\n",
        "vocab = sorted(counts, key=counts.get, reverse=True)\n",
        "vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n",
        "\n",
        "## use the dict to tokenize each review in reviews_split\n",
        "## store the tokenized reviews in reviews_ints\n",
        "reviews_ints = []\n",
        "for review in new_reviews:\n",
        "    reviews_ints.append([vocab_to_int[word] for word in review])"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z-4n9wTDxrdi",
        "outputId": "b867a61d-39f2-4601-e78b-57009f6eea75"
      },
      "source": [
        "# stats about vocabulary\n",
        "print('Unique words: ', len((vocab_to_int)))  # should ~ 74000+\n",
        "print()\n",
        "\n",
        "# print tokens in first review\n",
        "print('Tokenized review: \\n', reviews_ints[:1])"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique words:  16727\n",
            "\n",
            "Tokenized review: \n",
            " [[57, 213]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yj9wuiT62nZL"
      },
      "source": [
        "\n",
        "# Encoding the labels\n",
        "As mentioned before, our goal is to identify whether a tweet is negative or non-negative (positive or neutral). Our labels are \"positive\", \"negative\", or \"neutral. To use these labels in our network, we need to convert them to 0 and 1."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EKEgca0Tx0D6"
      },
      "source": [
        "# 2=positive, 1=neutral, 0=negative label conversion\n",
        "encoded_labels = []\n",
        "for label in labels:\n",
        "    if label == 'neutral':\n",
        "        encoded_labels.append(1)\n",
        "    elif label == 'negative':\n",
        "        encoded_labels.append(0)\n",
        "    else:\n",
        "        encoded_labels.append(1)\n",
        "\n",
        "encoded_labels = np.asarray(encoded_labels)"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ktbW4ArycNT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8fe8a044-33ea-4382-b571-23f98d8c49a9"
      },
      "source": [
        "encoded_labels"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, ..., 1, 0, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_INNAoPL2rTV"
      },
      "source": [
        "# Padding sequences\n",
        "To deal with both short and very long reviews, we'll pad or truncate all our reviews to a specific length. For reviews shorter than some seq_length, we'll pad with 0s. For reviews longer than seq_length, we can truncate them to the first seq_length words. A good seq_length, in this case, is 30, because the maximum review length from the data is 32."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dnva_A8gx6oq"
      },
      "source": [
        "def pad_features(reviews_ints, seq_length):\n",
        "    ''' Return features of review_ints, where each review is padded with 0's \n",
        "        or truncated to the input seq_length.\n",
        "    '''\n",
        "    \n",
        "    # getting the correct rows x cols shape\n",
        "    features = np.zeros((len(reviews_ints), seq_length), dtype=int)\n",
        "\n",
        "    # for each review, I grab that review and \n",
        "    for i, row in enumerate(reviews_ints):\n",
        "        features[i, -len(row):] = np.array(row)[:seq_length]\n",
        "    \n",
        "    return features"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DNfEPSGzx0Gy",
        "outputId": "20be638b-986f-4eec-e034-7312a7e32c62"
      },
      "source": [
        "# Test implementation!\n",
        "\n",
        "seq_length = 30\n",
        "\n",
        "features = pad_features(reviews_ints, seq_length=seq_length)\n",
        "\n",
        "## test statements \n",
        "assert len(features)==len(reviews_ints), \"The features should have as many rows as reviews.\"\n",
        "assert len(features[0])==seq_length, \"Each feature row should contain seq_length values.\"\n",
        "\n",
        "# print first 10 values of the first 30 batches \n",
        "print(features[:10,:10])"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0 446]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]\n",
            " [  0   0   0   0   0   0   0   0   0   0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wCMCgbdd22gk"
      },
      "source": [
        "# Training, validation, and test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-hhmNSp7x0I7",
        "outputId": "99ea30b5-aeb4-4658-da5b-81b43e68f704"
      },
      "source": [
        "\n",
        "split_frac = 0.8\n",
        "\n",
        "## split data into training, validation, and test data (features and labels, x and y)\n",
        "\n",
        "split_idx = int(len(features)*split_frac)\n",
        "train_x, remaining_x = features[:split_idx], features[split_idx:]\n",
        "train_y, remaining_y = encoded_labels[:split_idx], encoded_labels[split_idx:]\n",
        "\n",
        "test_idx = int(len(remaining_x)*0.5)\n",
        "val_x, test_x = remaining_x[:test_idx], remaining_x[test_idx:]\n",
        "val_y, test_y = remaining_y[:test_idx], remaining_y[test_idx:]\n",
        "\n",
        "## print out the shapes of the resultant feature data\n",
        "print(\"\\t\\t\\tFeature Shapes:\")\n",
        "print(\"Train set: \\t\\t{}\".format(train_x.shape), \n",
        "      \"\\nValidation set: \\t{}\".format(val_x.shape),\n",
        "      \"\\nTest set: \\t\\t{}\".format(test_x.shape))"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\t\t\tFeature Shapes:\n",
            "Train set: \t\t(11200, 30) \n",
            "Validation set: \t(1400, 30) \n",
            "Test set: \t\t(1400, 30)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S3HeBzAo261F"
      },
      "source": [
        "\n",
        "# DataLoaders and Batching\n",
        "After creating training, test, and validation data, we can create DataLoaders for this data by following two steps:\n",
        "\n",
        "Create a known format for accessing our data, using TensorDataset which takes in an input set of data and a target set of data with the same first dimension, and creates a dataset.\n",
        "Create DataLoaders and batch our training, validation, and test Tensor datasets."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "voYeWB-dyMUE"
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "\n",
        "# create Tensor datasets\n",
        "train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))\n",
        "valid_data = TensorDataset(torch.from_numpy(val_x), torch.from_numpy(val_y))\n",
        "test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))\n",
        "\n",
        "# dataloaders\n",
        "batch_size = 50\n",
        "\n",
        "# make sure the SHUFFLE the training data\n",
        "train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)\n",
        "valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)\n",
        "test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5w5NF1yryMWa",
        "outputId": "9dee28a2-1b1d-4ed7-e61a-88dca89ed2f5"
      },
      "source": [
        "# obtain one batch of training data\n",
        "dataiter = iter(train_loader)\n",
        "sample_x, sample_y = dataiter.next()\n",
        "\n",
        "print('Sample input size: ', sample_x.size()) # batch_size, seq_length\n",
        "print('Sample input: \\n', sample_x)\n",
        "print()\n",
        "print('Sample label size: ', sample_y.size()) # batch_size\n",
        "print('Sample label: \\n', sample_y)"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample input size:  torch.Size([50, 30])\n",
            "Sample input: \n",
            " tensor([[    0,     0,     0,  ...,   286,   937,  4032],\n",
            "        [    0,     0,     0,  ...,     4,   296,    83],\n",
            "        [    0,     0,     0,  ...,  1283,    52, 14510],\n",
            "        ...,\n",
            "        [    0,     0,     0,  ...,   169,  1008,    19],\n",
            "        [    0,     0,     0,  ...,   413,  3534,   692],\n",
            "        [    0,     0,     0,  ..., 13926,  1705,    35]])\n",
            "\n",
            "Sample label size:  torch.Size([50])\n",
            "Sample label: \n",
            " tensor([1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0,\n",
            "        1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n",
            "        0, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-m_PkBZyMZS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8e810555-dfda-4d4d-e2c6-455c79aa49da"
      },
      "source": [
        "# First checking if GPU is available\n",
        "train_on_gpu=torch.cuda.is_available()\n",
        "\n",
        "if(train_on_gpu):\n",
        "    print('Training on GPU.')\n",
        "else:\n",
        "    print('No GPU available, training on CPU.')"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on GPU.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hOSbEjOv3Hbs"
      },
      "source": [
        "# Sentiment Network with PyTorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqnApIibyMb6"
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class SentimentRNN(nn.Module):\n",
        "    \"\"\"\n",
        "    The RNN model that will be used to perform Sentiment analysis.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):\n",
        "        \"\"\"\n",
        "        Initialize the model by setting up the layers.\n",
        "        \"\"\"\n",
        "        super(SentimentRNN, self).__init__()\n",
        "\n",
        "        self.output_size = output_size\n",
        "        self.n_layers = n_layers\n",
        "        self.hidden_dim = hidden_dim\n",
        "        \n",
        "        # embedding and LSTM layers\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, \n",
        "                            dropout=drop_prob, batch_first=True)\n",
        "        \n",
        "        # dropout layer\n",
        "        self.dropout = nn.Dropout(0.3)\n",
        "        \n",
        "        # linear and sigmoid layers\n",
        "        self.fc = nn.Linear(hidden_dim, output_size)\n",
        "        self.sig = nn.Sigmoid()\n",
        "        \n",
        "\n",
        "    def forward(self, x, hidden):\n",
        "        \"\"\"\n",
        "        Perform a forward pass of our model on some input and hidden state.\n",
        "        \"\"\"\n",
        "        batch_size = x.size(0)\n",
        "\n",
        "        # embeddings and lstm_out\n",
        "        x = x.long()\n",
        "        embeds = self.embedding(x)\n",
        "        lstm_out, hidden = self.lstm(embeds, hidden)\n",
        "    \n",
        "        # stack up lstm outputs\n",
        "        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
        "        \n",
        "        # dropout and fully-connected layer\n",
        "        out = self.dropout(lstm_out)\n",
        "        out = self.fc(out)\n",
        "        # sigmoid function\n",
        "        sig_out = self.sig(out)\n",
        "        \n",
        "        # reshape to be batch_size first\n",
        "        sig_out = sig_out.view(batch_size, -1)\n",
        "        sig_out = sig_out[:, -1] # get last batch of labels\n",
        "        \n",
        "        # return last sigmoid output and hidden state\n",
        "        return sig_out, hidden\n",
        "    \n",
        "    \n",
        "    def init_hidden(self, batch_size):\n",
        "        ''' Initializes hidden state '''\n",
        "        # Create two new tensors with sizes n_layers x batch_size x hidden_dim,\n",
        "        # initialized to zero, for hidden state and cell state of LSTM\n",
        "        weight = next(self.parameters()).data\n",
        "        \n",
        "        if (train_on_gpu):\n",
        "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
        "                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
        "        else:\n",
        "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),\n",
        "                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())\n",
        "        \n",
        "        return hidden"
      ],
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oJDEN8w11M_T",
        "outputId": "e61fcf66-0b35-4fd2-944e-7d945c6ad0fc"
      },
      "source": [
        "\n",
        "# Instantiate the model w/ hyperparams\n",
        "vocab_size = len(vocab_to_int)+1 # +1 for the 0 padding + our word tokens\n",
        "output_size = 1\n",
        "embedding_dim = 200\n",
        "hidden_dim = 128\n",
        "n_layers = 2\n",
        "\n",
        "net = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)\n",
        "\n",
        "print(net)"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SentimentRNN(\n",
            "  (embedding): Embedding(16728, 200)\n",
            "  (lstm): LSTM(200, 128, num_layers=2, batch_first=True, dropout=0.5)\n",
            "  (dropout): Dropout(p=0.3, inplace=False)\n",
            "  (fc): Linear(in_features=128, out_features=1, bias=True)\n",
            "  (sig): Sigmoid()\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9FP804k1NIb"
      },
      "source": [
        "# loss and optimization functions\n",
        "lr=0.001\n",
        "\n",
        "criterion = nn.BCELoss()\n",
        "optimizer = torch.optim.Adam(net.parameters(), lr=lr)"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GiWZXcJd1NLU",
        "outputId": "6974b6d2-0c69-47c9-caa1-f72d9d3eea5e"
      },
      "source": [
        "# training params\n",
        "\n",
        "epochs = 20 \n",
        "\n",
        "counter = 0\n",
        "print_every = 100\n",
        "clip=5 # gradient clipping\n",
        "\n",
        "# move model to GPU, if available\n",
        "if(train_on_gpu):\n",
        "    net.cuda()\n",
        "\n",
        "net.train()\n",
        "# train for some number of epochs\n",
        "for e in range(epochs):\n",
        "    # initialize hidden state\n",
        "    h = net.init_hidden(batch_size)\n",
        "\n",
        "    # batch loop\n",
        "    for inputs, labels in train_loader:\n",
        "        counter += 1\n",
        "\n",
        "        if(train_on_gpu):\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "\n",
        "        # Creating new variables for the hidden state, otherwise\n",
        "        # we'd backprop through the entire training history\n",
        "        h = tuple([each.data for each in h])\n",
        "\n",
        "        # zero accumulated gradients\n",
        "        net.zero_grad()\n",
        "\n",
        "        # get the output from the model\n",
        "        output, h = net(inputs, h)\n",
        "\n",
        "        # calculate the loss and perform backprop\n",
        "        loss = criterion(output.squeeze(), labels.float())\n",
        "        loss.backward()\n",
        "        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
        "        nn.utils.clip_grad_norm_(net.parameters(), clip)\n",
        "        optimizer.step()\n",
        "\n",
        "        # loss stats\n",
        "        if counter % print_every == 0:\n",
        "            # Get validation loss\n",
        "            val_h = net.init_hidden(batch_size)\n",
        "            val_losses = []\n",
        "            net.eval()\n",
        "            for inputs, labels in valid_loader:\n",
        "\n",
        "                # Creating new variables for the hidden state, otherwise\n",
        "                # we'd backprop through the entire training history\n",
        "                val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "                if(train_on_gpu):\n",
        "                    inputs, labels = inputs.cuda(), labels.cuda()\n",
        "\n",
        "                output, val_h = net(inputs, val_h)\n",
        "                val_loss = criterion(output.squeeze(), labels.float())\n",
        "\n",
        "                val_losses.append(val_loss.item())\n",
        "\n",
        "            net.train()\n",
        "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
        "                  \"Step: {}...\".format(counter),\n",
        "                  \"Loss: {:.6f}...\".format(loss.item()),\n",
        "                  \"Val Loss: {:.6f}\".format(np.mean(val_losses)))"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1/20... Step: 100... Loss: 0.536182... Val Loss: 0.494789\n",
            "Epoch: 1/20... Step: 200... Loss: 0.342242... Val Loss: 0.481267\n",
            "Epoch: 2/20... Step: 300... Loss: 0.424031... Val Loss: 0.430037\n",
            "Epoch: 2/20... Step: 400... Loss: 0.384714... Val Loss: 0.436275\n",
            "Epoch: 3/20... Step: 500... Loss: 0.344898... Val Loss: 0.512167\n",
            "Epoch: 3/20... Step: 600... Loss: 0.211354... Val Loss: 0.443102\n",
            "Epoch: 4/20... Step: 700... Loss: 0.218554... Val Loss: 0.540556\n",
            "Epoch: 4/20... Step: 800... Loss: 0.282090... Val Loss: 0.538329\n",
            "Epoch: 5/20... Step: 900... Loss: 0.116927... Val Loss: 0.568285\n",
            "Epoch: 5/20... Step: 1000... Loss: 0.091082... Val Loss: 0.586823\n",
            "Epoch: 5/20... Step: 1100... Loss: 0.064183... Val Loss: 0.673234\n",
            "Epoch: 6/20... Step: 1200... Loss: 0.100836... Val Loss: 0.820781\n",
            "Epoch: 6/20... Step: 1300... Loss: 0.029550... Val Loss: 0.851291\n",
            "Epoch: 7/20... Step: 1400... Loss: 0.007672... Val Loss: 0.840712\n",
            "Epoch: 7/20... Step: 1500... Loss: 0.017947... Val Loss: 0.887218\n",
            "Epoch: 8/20... Step: 1600... Loss: 0.064415... Val Loss: 0.852542\n",
            "Epoch: 8/20... Step: 1700... Loss: 0.005255... Val Loss: 0.983539\n",
            "Epoch: 9/20... Step: 1800... Loss: 0.013786... Val Loss: 0.926213\n",
            "Epoch: 9/20... Step: 1900... Loss: 0.010355... Val Loss: 0.969622\n",
            "Epoch: 9/20... Step: 2000... Loss: 0.037333... Val Loss: 0.965913\n",
            "Epoch: 10/20... Step: 2100... Loss: 0.006790... Val Loss: 1.071350\n",
            "Epoch: 10/20... Step: 2200... Loss: 0.041043... Val Loss: 0.992211\n",
            "Epoch: 11/20... Step: 2300... Loss: 0.012228... Val Loss: 0.994055\n",
            "Epoch: 11/20... Step: 2400... Loss: 0.006787... Val Loss: 1.101592\n",
            "Epoch: 12/20... Step: 2500... Loss: 0.003977... Val Loss: 1.040750\n",
            "Epoch: 12/20... Step: 2600... Loss: 0.051130... Val Loss: 1.006338\n",
            "Epoch: 13/20... Step: 2700... Loss: 0.023056... Val Loss: 1.045345\n",
            "Epoch: 13/20... Step: 2800... Loss: 0.000416... Val Loss: 1.193788\n",
            "Epoch: 13/20... Step: 2900... Loss: 0.005105... Val Loss: 1.229952\n",
            "Epoch: 14/20... Step: 3000... Loss: 0.003819... Val Loss: 1.195661\n",
            "Epoch: 14/20... Step: 3100... Loss: 0.038420... Val Loss: 1.150890\n",
            "Epoch: 15/20... Step: 3200... Loss: 0.001121... Val Loss: 1.111401\n",
            "Epoch: 15/20... Step: 3300... Loss: 0.004511... Val Loss: 1.114257\n",
            "Epoch: 16/20... Step: 3400... Loss: 0.002109... Val Loss: 1.138698\n",
            "Epoch: 16/20... Step: 3500... Loss: 0.001940... Val Loss: 1.292685\n",
            "Epoch: 17/20... Step: 3600... Loss: 0.015803... Val Loss: 1.122164\n",
            "Epoch: 17/20... Step: 3700... Loss: 0.001603... Val Loss: 1.134488\n",
            "Epoch: 17/20... Step: 3800... Loss: 0.173662... Val Loss: 1.122638\n",
            "Epoch: 18/20... Step: 3900... Loss: 0.016160... Val Loss: 1.115329\n",
            "Epoch: 18/20... Step: 4000... Loss: 0.003738... Val Loss: 1.257620\n",
            "Epoch: 19/20... Step: 4100... Loss: 0.001353... Val Loss: 1.307187\n",
            "Epoch: 19/20... Step: 4200... Loss: 0.008524... Val Loss: 1.519380\n",
            "Epoch: 20/20... Step: 4300... Loss: 0.004698... Val Loss: 1.343439\n",
            "Epoch: 20/20... Step: 4400... Loss: 0.018503... Val Loss: 1.413271\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SZ_ysPZy1NNz",
        "outputId": "79cb0d76-89bc-476a-e929-0e446fa6b2e1"
      },
      "source": [
        "# Get test data loss and accuracy\n",
        "\n",
        "test_losses = [] # track loss\n",
        "num_correct = 0\n",
        "\n",
        "# init hidden state\n",
        "h = net.init_hidden(batch_size)\n",
        "\n",
        "net.eval()\n",
        "# iterate over test data\n",
        "for inputs, labels in test_loader:\n",
        "\n",
        "    # Creating new variables for the hidden state, otherwise\n",
        "    # we'd backprop through the entire training history\n",
        "    h = tuple([each.data for each in h])\n",
        "\n",
        "    if(train_on_gpu):\n",
        "        inputs, labels = inputs.cuda(), labels.cuda()\n",
        "    \n",
        "    # get predicted outputs\n",
        "    output, h = net(inputs, h)\n",
        "    \n",
        "    # calculate loss\n",
        "    test_loss = criterion(output.squeeze(), labels.float())\n",
        "    test_losses.append(test_loss.item())\n",
        "    \n",
        "    # convert output probabilities to predicted class (0 or 1)\n",
        "    pred = torch.round(output.squeeze())  # rounds to the nearest integer\n",
        "    \n",
        "    # compare predictions to true label\n",
        "    correct_tensor = pred.eq(labels.float().view_as(pred))\n",
        "    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())\n",
        "    num_correct += np.sum(correct)\n",
        "\n",
        "\n",
        "# -- stats! -- ##\n",
        "# avg test loss\n",
        "print(\"Test loss: {:.3f}\".format(np.mean(test_losses)))\n",
        "\n",
        "# accuracy over all test data\n",
        "test_acc = num_correct/len(test_loader.dataset)\n",
        "print(\"Test accuracy: {:.3f}\".format(test_acc))"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 1.150\n",
            "Test accuracy: 0.837\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AUdxmsOH1NQi"
      },
      "source": [
        ""
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "reS8A6E21NTT"
      },
      "source": [
        ""
      ],
      "execution_count": 68,
      "outputs": []
    }
  ]
}