Classifying_Partisianship_Using_Deep_Learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Classifying Partisianship Using Deep Learning.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMoV8d0/weXODG2pY4pcH8Q",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/prishanmu/SI670_FinalProject/blob/master/Classifying_Partisianship_Using_Deep_Learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dalUafHObcef",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data proces1sing, CSV file I/O (e.g. pd.read_csv)\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.classify.scikitlearn import SklearnClassifier\n",
        "from sklearn import model_selection\n",
        "import nltk\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from wordcloud import WordCloud\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f-N3TBW_eTQD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.python import keras\n",
        "from tensorflow.python.keras.models import Sequential\n",
        "from tensorflow.python.keras.layers import Dense, Flatten, Conv2D, Dropout"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JiVDUqUBbu3q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 189
        },
        "outputId": "b5475160-1ad2-4625-9008-32f66edc7501"
      },
      "source": [
        "data = pd.read_csv(\"political_social_media.csv\", encoding = \"ISO-8859-1\")\n",
        "data = data.drop(columns = ['_golden', '_unit_state', 'orig__golden', 'audience_gold', 'embed', 'message_gold', '_last_judgment_at', 'bias_gold', 'id', '_unit_id', '_trusted_judgments'])\n",
        "data.head()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>audience</th>\n",
              "      <th>audience:confidence</th>\n",
              "      <th>bias</th>\n",
              "      <th>bias:confidence</th>\n",
              "      <th>message</th>\n",
              "      <th>message:confidence</th>\n",
              "      <th>bioid</th>\n",
              "      <th>label</th>\n",
              "      <th>source</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>national</td>\n",
              "      <td>1.0</td>\n",
              "      <td>partisan</td>\n",
              "      <td>1.0</td>\n",
              "      <td>policy</td>\n",
              "      <td>1.0</td>\n",
              "      <td>R000596</td>\n",
              "      <td>From: Trey Radel (Representative from Florida)</td>\n",
              "      <td>twitter</td>\n",
              "      <td>RT @nowthisnews: Rep. Trey Radel (R- #FL) slam...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>national</td>\n",
              "      <td>1.0</td>\n",
              "      <td>partisan</td>\n",
              "      <td>1.0</td>\n",
              "      <td>attack</td>\n",
              "      <td>1.0</td>\n",
              "      <td>M000355</td>\n",
              "      <td>From: Mitch McConnell (Senator from Kentucky)</td>\n",
              "      <td>twitter</td>\n",
              "      <td>VIDEO - #Obamacare:  Full of Higher Costs and ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>national</td>\n",
              "      <td>1.0</td>\n",
              "      <td>neutral</td>\n",
              "      <td>1.0</td>\n",
              "      <td>support</td>\n",
              "      <td>1.0</td>\n",
              "      <td>S001180</td>\n",
              "      <td>From: Kurt Schrader (Representative from Oregon)</td>\n",
              "      <td>twitter</td>\n",
              "      <td>Please join me today in remembering our fallen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>national</td>\n",
              "      <td>1.0</td>\n",
              "      <td>neutral</td>\n",
              "      <td>1.0</td>\n",
              "      <td>policy</td>\n",
              "      <td>1.0</td>\n",
              "      <td>C000880</td>\n",
              "      <td>From: Michael Crapo (Senator from Idaho)</td>\n",
              "      <td>twitter</td>\n",
              "      <td>RT @SenatorLeahy: 1st step toward Senate debat...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>national</td>\n",
              "      <td>1.0</td>\n",
              "      <td>partisan</td>\n",
              "      <td>1.0</td>\n",
              "      <td>policy</td>\n",
              "      <td>1.0</td>\n",
              "      <td>U000038</td>\n",
              "      <td>From: Mark Udall (Senator from Colorado)</td>\n",
              "      <td>twitter</td>\n",
              "      <td>.@amazon delivery #drones show need to update ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   audience  ...                                               text\n",
              "0  national  ...  RT @nowthisnews: Rep. Trey Radel (R- #FL) slam...\n",
              "1  national  ...  VIDEO - #Obamacare:  Full of Higher Costs and ...\n",
              "2  national  ...  Please join me today in remembering our fallen...\n",
              "3  national  ...  RT @SenatorLeahy: 1st step toward Senate debat...\n",
              "4  national  ...  .@amazon delivery #drones show need to update ...\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NKwzvso5bxek",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "ac937ee1-ac66-4276-adf1-eb3656fcac8b"
      },
      "source": [
        "##########################\n",
        "### FILTERING & ENCODING\n",
        "##########################\n",
        "\n",
        "###Filter out any rows with less than confidence than 1 for any measure\n",
        "data = data[data['audience:confidence'] == 1.0]\n",
        "data = data[data['bias:confidence'] == 1.0]\n",
        "data = data[data['message:confidence'] == 1.0]\n",
        "data = data.drop(['audience:confidence', 'bias:confidence','message:confidence' ], axis=1)\n",
        "\n",
        "###Split and extract info from label column\n",
        "data[['pol_name','pol_info']] = data.label.str.split(\"\\(\",expand=True) \n",
        "from_split = data.pol_info.str.split(\"from\",expand=True) \n",
        "from_split[1] = from_split[1].str.replace(r'\\)$', '')\n",
        "data['pol_type'] = from_split[0]\n",
        "data['pol_state'] = from_split[1]\n",
        "data = data.drop(['label', 'pol_name','pol_info' ], axis=1)\n",
        "\n",
        "###Encode messages and pol_state with get_dummies\n",
        "data = pd.concat([data,pd.get_dummies(data['message'], prefix='message')],axis=1).drop(['message'],axis=1)\n",
        "data = pd.concat([data,pd.get_dummies(data['pol_state'], prefix='pol_state')],axis=1).drop(['pol_state'],axis=1)\n",
        "\n",
        "###Encode pol_type, bioid, audience, bias, and source isth LabelEncoder\n",
        "encoder = LabelEncoder()\n",
        "\n",
        "pol_type_encoded = encoder.fit_transform(data['pol_type'])\n",
        "data['pol_type'] = pol_type_encoded\n",
        "\n",
        "audience_encoded = encoder.fit_transform(data['audience'])\n",
        "data['audience'] = audience_encoded\n",
        "\n",
        "bias_encoded = encoder.fit_transform(data['bias'])\n",
        "data['bias'] = bias_encoded\n",
        "\n",
        "source_encoded = encoder.fit_transform(data['source'])\n",
        "data['source'] = source_encoded\n",
        "\n",
        "bioid_encoded = encoder.fit_transform(data['bioid'])\n",
        "data['bioid'] = bioid_encoded\n",
        "\n",
        "data.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>audience</th>\n",
              "      <th>bias</th>\n",
              "      <th>bioid</th>\n",
              "      <th>source</th>\n",
              "      <th>text</th>\n",
              "      <th>pol_type</th>\n",
              "      <th>message_attack</th>\n",
              "      <th>message_constituency</th>\n",
              "      <th>message_information</th>\n",
              "      <th>message_media</th>\n",
              "      <th>message_mobilization</th>\n",
              "      <th>message_other</th>\n",
              "      <th>message_personal</th>\n",
              "      <th>message_policy</th>\n",
              "      <th>message_support</th>\n",
              "      <th>pol_state_ Alabama</th>\n",
              "      <th>pol_state_ Alaska</th>\n",
              "      <th>pol_state_ Arizona</th>\n",
              "      <th>pol_state_ Arkansas</th>\n",
              "      <th>pol_state_ California</th>\n",
              "      <th>pol_state_ Colorado</th>\n",
              "      <th>pol_state_ Connecticut</th>\n",
              "      <th>pol_state_ Delaware</th>\n",
              "      <th>pol_state_ Florida</th>\n",
              "      <th>pol_state_ Georgia</th>\n",
              "      <th>pol_state_ Hawaii</th>\n",
              "      <th>pol_state_ Idaho</th>\n",
              "      <th>pol_state_ Illinois</th>\n",
              "      <th>pol_state_ Indiana</th>\n",
              "      <th>pol_state_ Iowa</th>\n",
              "      <th>pol_state_ Kansas</th>\n",
              "      <th>pol_state_ Kentucky</th>\n",
              "      <th>pol_state_ Louisiana</th>\n",
              "      <th>pol_state_ Maine</th>\n",
              "      <th>pol_state_ Maryland</th>\n",
              "      <th>pol_state_ Massachusetts</th>\n",
              "      <th>pol_state_ Michigan</th>\n",
              "      <th>pol_state_ Minnesota</th>\n",
              "      <th>pol_state_ Mississippi</th>\n",
              "      <th>pol_state_ Missouri</th>\n",
              "      <th>pol_state_ Montana</th>\n",
              "      <th>pol_state_ NA</th>\n",
              "      <th>pol_state_ Nebraska</th>\n",
              "      <th>pol_state_ Nevada</th>\n",
              "      <th>pol_state_ New Hampshire</th>\n",
              "      <th>pol_state_ New Jersey</th>\n",
              "      <th>pol_state_ New Mexico</th>\n",
              "      <th>pol_state_ New York</th>\n",
              "      <th>pol_state_ North Carolina</th>\n",
              "      <th>pol_state_ North Dakota</th>\n",
              "      <th>pol_state_ Ohio</th>\n",
              "      <th>pol_state_ Oklahoma</th>\n",
              "      <th>pol_state_ Oregon</th>\n",
              "      <th>pol_state_ Pennsylvania</th>\n",
              "      <th>pol_state_ Rhode Island</th>\n",
              "      <th>pol_state_ South Carolina</th>\n",
              "      <th>pol_state_ South Dakota</th>\n",
              "      <th>pol_state_ Tennessee</th>\n",
              "      <th>pol_state_ Texas</th>\n",
              "      <th>pol_state_ Utah</th>\n",
              "      <th>pol_state_ Vermont</th>\n",
              "      <th>pol_state_ Virginia</th>\n",
              "      <th>pol_state_ Washington</th>\n",
              "      <th>pol_state_ West Virginia</th>\n",
              "      <th>pol_state_ Wisconsin</th>\n",
              "      <th>pol_state_ Wyoming</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>396</td>\n",
              "      <td>1</td>\n",
              "      <td>RT @nowthisnews: Rep. Trey Radel (R- #FL) slam...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>289</td>\n",
              "      <td>1</td>\n",
              "      <td>VIDEO - #Obamacare:  Full of Higher Costs and ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>430</td>\n",
              "      <td>1</td>\n",
              "      <td>Please join me today in remembering our fallen...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>63</td>\n",
              "      <td>1</td>\n",
              "      <td>RT @SenatorLeahy: 1st step toward Senate debat...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>462</td>\n",
              "      <td>1</td>\n",
              "      <td>.@amazon delivery #drones show need to update ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   audience  bias  ...  pol_state_ Wisconsin  pol_state_ Wyoming\n",
              "0         1     1  ...                     0                   0\n",
              "1         1     1  ...                     0                   0\n",
              "2         1     0  ...                     0                   0\n",
              "3         1     0  ...                     0                   0\n",
              "4         1     1  ...                     0                   0\n",
              "\n",
              "[5 rows x 66 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fllRdBsLb9xS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "f6f81244-20f1-40c2-dd1a-8f1fbc25be81"
      },
      "source": [
        "########################\n",
        "### TEXT PRE-PROCESSING\n",
        "########################\n",
        "\n",
        "#Using regular expressions to remove URLs, numbers etc\n",
        "processed = data['text'].str.replace(r'^.+@[^\\.].*\\.[a-z]{2,}$',' ')\n",
        "processed = processed.str.replace(r'^http\\://[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*)?$',' ')\n",
        "processed = processed.str.replace(r'http',' ')\n",
        "processed = processed.str.replace(r'£|\\$', ' ')\n",
        "processed = processed.str.replace(r'^\\(?[\\d]{3}\\)?[\\s-]?[\\d]{3}[\\s-]?[\\d]{4}$',' ')\n",
        "processed = processed.str.replace(r'\\d+(\\.\\d+)?', ' ')\n",
        "processed = processed.str.replace(r'[^\\w\\d\\s]', ' ')\n",
        "processed = processed.str.replace(r'\\s+', ' ')\n",
        "processed = processed.str.replace(r'^\\s+|\\s+?$', ' ')\n",
        "processed = processed.str.lower()\n",
        "\n",
        "#Removing stop words from text\n",
        "stop_words = set(stopwords.words('english'))\n",
        "processed = processed.apply(lambda x: ' '.join(\n",
        "    term for term in x.split() if term not in stop_words))\n",
        "\n",
        "#Removing meaningless words from text\n",
        "processed = processed.apply(lambda x: ' '.join(\n",
        "    term for term in x.split() if term not in ['ûªs', 'û', 'ûªt', 'r', 'en', 'fl', 'p', 'va', 'amp', 'icymi', 'th', 'pm', 'hours', 'u']))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "error",
          "ename": "LookupError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mLookupError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}/{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/data.py\u001b[0m in \u001b[0;36mfind\u001b[0;34m(resource_name, paths)\u001b[0m\n\u001b[1;32m    672\u001b[0m     \u001b[0mresource_not_found\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'\\n%s\\n%s\\n%s\\n'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource_not_found\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mstopwords\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('stopwords')\n  \u001b[0m\n  Searched in:\n    - '/root/nltk_data'\n    - '/usr/share/nltk_data'\n    - '/usr/local/share/nltk_data'\n    - '/usr/lib/nltk_data'\n    - '/usr/local/lib/nltk_data'\n    - '/usr/nltk_data'\n    - '/usr/lib/nltk_data'\n**********************************************************************\n",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mLookupError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-ee8fa07569a9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;31m#Removing stop words from text\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mstop_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m processed = processed.apply(lambda x: ' '.join(\n\u001b[1;32m     20\u001b[0m     term for term in x.split() if term not in stop_words))\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, attr)\u001b[0m\n\u001b[1;32m    114\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"LazyCorpusLoader object has no attribute '__bases__'\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m         \u001b[0;31m# This looks circular, but its not, since __load() changes our\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0;31m# __class__ to something new:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}/{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m                 \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m         \u001b[0;31m# Load the corpus.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/corpus/util.py\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     76\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m                 \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}/{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}/{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/data.py\u001b[0m in \u001b[0;36mfind\u001b[0;34m(resource_name, paths)\u001b[0m\n\u001b[1;32m    671\u001b[0m     \u001b[0msep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'*'\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m70\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m     \u001b[0mresource_not_found\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'\\n%s\\n%s\\n%s\\n'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource_not_found\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mstopwords\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('stopwords')\n  \u001b[0m\n  Searched in:\n    - '/root/nltk_data'\n    - '/usr/share/nltk_data'\n    - '/usr/local/share/nltk_data'\n    - '/usr/lib/nltk_data'\n    - '/usr/local/lib/nltk_data'\n    - '/usr/nltk_data'\n    - '/usr/lib/nltk_data'\n**********************************************************************\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bDXZRDeRb9lU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 254
        },
        "outputId": "44334014-fb0e-48ae-d78f-5bd170575079"
      },
      "source": [
        "########################\n",
        "### TEXT PROCESSING\n",
        "########################\n",
        "\n",
        "MAXFEATURES = 2000\n",
        "\n",
        "tfidf_vect = TfidfVectorizer(max_features = MAXFEATURES)\n",
        "\n",
        "text_tfidf = tfidf_vect.fit_transform(data['text'])\n",
        "text_tfidf = pd.DataFrame(text_tfidf.toarray())\n",
        "\n",
        "data = pd.concat([data,text_tfidf],axis=1).drop(['text'],axis=1)\n",
        "data.head()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>audience</th>\n",
              "      <th>bias</th>\n",
              "      <th>bioid</th>\n",
              "      <th>source</th>\n",
              "      <th>pol_type</th>\n",
              "      <th>message_attack</th>\n",
              "      <th>message_constituency</th>\n",
              "      <th>message_information</th>\n",
              "      <th>message_media</th>\n",
              "      <th>message_mobilization</th>\n",
              "      <th>message_other</th>\n",
              "      <th>message_personal</th>\n",
              "      <th>message_policy</th>\n",
              "      <th>message_support</th>\n",
              "      <th>pol_state_ Alabama</th>\n",
              "      <th>pol_state_ Alaska</th>\n",
              "      <th>pol_state_ Arizona</th>\n",
              "      <th>pol_state_ Arkansas</th>\n",
              "      <th>pol_state_ California</th>\n",
              "      <th>pol_state_ Colorado</th>\n",
              "      <th>pol_state_ Connecticut</th>\n",
              "      <th>pol_state_ Delaware</th>\n",
              "      <th>pol_state_ Florida</th>\n",
              "      <th>pol_state_ Georgia</th>\n",
              "      <th>pol_state_ Hawaii</th>\n",
              "      <th>pol_state_ Idaho</th>\n",
              "      <th>pol_state_ Illinois</th>\n",
              "      <th>pol_state_ Indiana</th>\n",
              "      <th>pol_state_ Iowa</th>\n",
              "      <th>pol_state_ Kansas</th>\n",
              "      <th>pol_state_ Kentucky</th>\n",
              "      <th>pol_state_ Louisiana</th>\n",
              "      <th>pol_state_ Maine</th>\n",
              "      <th>pol_state_ Maryland</th>\n",
              "      <th>pol_state_ Massachusetts</th>\n",
              "      <th>pol_state_ Michigan</th>\n",
              "      <th>pol_state_ Minnesota</th>\n",
              "      <th>pol_state_ Mississippi</th>\n",
              "      <th>pol_state_ Missouri</th>\n",
              "      <th>pol_state_ Montana</th>\n",
              "      <th>...</th>\n",
              "      <th>1960</th>\n",
              "      <th>1961</th>\n",
              "      <th>1962</th>\n",
              "      <th>1963</th>\n",
              "      <th>1964</th>\n",
              "      <th>1965</th>\n",
              "      <th>1966</th>\n",
              "      <th>1967</th>\n",
              "      <th>1968</th>\n",
              "      <th>1969</th>\n",
              "      <th>1970</th>\n",
              "      <th>1971</th>\n",
              "      <th>1972</th>\n",
              "      <th>1973</th>\n",
              "      <th>1974</th>\n",
              "      <th>1975</th>\n",
              "      <th>1976</th>\n",
              "      <th>1977</th>\n",
              "      <th>1978</th>\n",
              "      <th>1979</th>\n",
              "      <th>1980</th>\n",
              "      <th>1981</th>\n",
              "      <th>1982</th>\n",
              "      <th>1983</th>\n",
              "      <th>1984</th>\n",
              "      <th>1985</th>\n",
              "      <th>1986</th>\n",
              "      <th>1987</th>\n",
              "      <th>1988</th>\n",
              "      <th>1989</th>\n",
              "      <th>1990</th>\n",
              "      <th>1991</th>\n",
              "      <th>1992</th>\n",
              "      <th>1993</th>\n",
              "      <th>1994</th>\n",
              "      <th>1995</th>\n",
              "      <th>1996</th>\n",
              "      <th>1997</th>\n",
              "      <th>1998</th>\n",
              "      <th>1999</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>396.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>289.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>430.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>63.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.158558</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>462.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 2065 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   audience  bias  bioid  source  pol_type  ...  1995  1996  1997  1998  1999\n",
              "0       1.0   1.0  396.0     1.0       0.0  ...   0.0   0.0   0.0   0.0   0.0\n",
              "1       1.0   1.0  289.0     1.0       1.0  ...   0.0   0.0   0.0   0.0   0.0\n",
              "2       1.0   0.0  430.0     1.0       0.0  ...   0.0   0.0   0.0   0.0   0.0\n",
              "3       1.0   0.0   63.0     1.0       1.0  ...   0.0   0.0   0.0   0.0   0.0\n",
              "4       1.0   1.0  462.0     1.0       1.0  ...   0.0   0.0   0.0   0.0   0.0\n",
              "\n",
              "[5 rows x 2065 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qeFgQ0dwcIra",
        "colab_type": "text"
      },
      "source": [
        "# Priya's Work: Deep Learning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1LkbcZT0m84M",
        "colab_type": "text"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "qxl1UH6km84M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = data.dropna()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "tShgUbIom84R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train-test split\n",
        "from sklearn.model_selection import train_test_split\n",
        "X = data.drop(columns = ['bias'])\n",
        "y = data['bias']\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)\n",
        "\n",
        "num_classes = 2"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AmCXt2LQcYXO",
        "colab_type": "text"
      },
      "source": [
        "## Model 1: Basic Sequential Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HL-MkWFcYAE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Conv2D(20, kernel_size=(3, 3),\n",
        "                 activation='relu',\n",
        "                 input_shape=(3582, 2064, 1)))\n",
        "model.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dense(num_classes, activation='softmax'))\n",
        "\n",
        "model.compile(loss=keras.losses.categorical_crossentropy,\n",
        "              optimizer='adam',\n",
        "              metrics=['accuracy'])\n",
        "model.fit(X, y,\n",
        "          batch_size=128,\n",
        "          epochs=2,\n",
        "          validation_split = 0.2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fA4YtlX8fM6p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# evaluation"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}