NLP_algorithms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing, naive_bayes, svm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset for positive and negative Amazon reviews  \n",
    "Downloaded from Github: https://gist.github.com/kunalj101/ad1d9c58d338e20d09ff26bcc06c4235  \n",
    "The dataset contains text with reviews for products boughts, as well as a label for this review.  \n",
    "__label__2 represents positive reviews, while __label__1 represents negative reviews  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = open('C:/Users/anna/Downloads/amazon/amazon_data/corpus', encoding=\"utf8\").read()\n",
    "labels, texts = [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, line in enumerate(data.split(\"\\n\")):\n",
    "    l = line.split()\n",
    "    labels.append(l[0])\n",
    "    texts.append(\" \".join(l[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a datafrme to store text and label\n",
    "data = pd.DataFrame()\n",
    "data['text'] = texts\n",
    "data['label'] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Stuning even for the non-gamer: This sound tra...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The best soundtrack ever to anything.: I'm rea...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Amazing!: This soundtrack is my favorite music...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Excellent Soundtrack: I truly like this soundt...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Remember, Pull Your Jaw Off The Floor After He...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>an absolute masterpiece: I am quite sure any o...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Buyer beware: This is a self-published book, a...</td>\n",
       "      <td>__label__1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Glorious story: I loved Whisper of the wicked ...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>A FIVE STAR BOOK: I just finished reading Whis...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Whispers of the Wicked Saints: This was a easy...</td>\n",
       "      <td>__label__2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text       label\n",
       "0  Stuning even for the non-gamer: This sound tra...  __label__2\n",
       "1  The best soundtrack ever to anything.: I'm rea...  __label__2\n",
       "2  Amazing!: This soundtrack is my favorite music...  __label__2\n",
       "3  Excellent Soundtrack: I truly like this soundt...  __label__2\n",
       "4  Remember, Pull Your Jaw Off The Floor After He...  __label__2\n",
       "5  an absolute masterpiece: I am quite sure any o...  __label__2\n",
       "6  Buyer beware: This is a self-published book, a...  __label__1\n",
       "7  Glorious story: I loved Whisper of the wicked ...  __label__2\n",
       "8  A FIVE STAR BOOK: I just finished reading Whis...  __label__2\n",
       "9  Whispers of the Wicked Saints: This was a easy...  __label__2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_size = 0.2  # we define 80% train - 20% test\n",
    "X_train, X_test, y_train, y_test = train_test_split(data['text'], data['label'], test_size=test_size, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As the output gets 2 values, we hot-encode those values\n",
    "y_train = preprocessing.LabelEncoder().fit_transform(y_train)\n",
    "y_test = preprocessing.LabelEncoder().fit_transform(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature extraction methods: \n",
    "Bags of words, Tf-Idf, n-grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bags-of-words\n",
    "The simplest and most intuitive NLP method , is the “bags-of-words”.\n",
    "It ignores sentence structure and simply counts how often each word occurs.   \n",
    "CountVectorizer allows us to use the bags-of-words approach, by converting a collection of text documents into a matrix of token counts.\n",
    "We instantiate the CountVectorizer and fit it to our training data, converting our collection of text documents into a matrix of token counts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# extracting words (at least 2 letters or numbers), then converts everything to lowercase \n",
    "# and builds a vocabulary using these tokens\n",
    "vect1 = CountVectorizer().fit(X_train)\n",
    "# get the built vocabularies\n",
    "features = vect1.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28031\n"
     ]
    }
   ],
   "source": [
    "# size of the built vocabulary\n",
    "num_of_feat = len(features)\n",
    "print(num_of_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform the documents of X_train to a document term matrix, which gives us the bags-of-word representation of X_train\n",
    "X_train_vectorized1 = vect1.transform(X_train)\n",
    "# The entries in this matrix are the number of times each word appears in each document. \n",
    "# Because the number of words in the vocabulary is so much larger than the number of words that might appear in a single text, \n",
    "# most entries of this matrix are zero.\n",
    "X_train_vectorized1.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tf-idf\n",
    "Tf-idf allows us to weight terms based on how important they are to a document.  \n",
    "For example, in a large text corpus, some words will be present very often but will carry very little meaningful information about the actual contents of the document. Those can be words such as 'the', 'a', 'I', 'is', 'are' etc.\n",
    "So, we will instantiate the tf–idf vectorizer and fit it to our training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8092\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# minimum document frequency min_df\n",
    "# keeps words that only appear in this number of documents we define\n",
    "min_df = 4  \n",
    "vect2 = TfidfVectorizer(min_df = min_df).fit(X_train)\n",
    "\n",
    "# now we have a new number of features\n",
    "feat2 = vect2.get_feature_names()\n",
    "num_of_feat2 = len(feat2)\n",
    "print(num_of_feat2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_vectorized2 = vect2.transform(X_train)\n",
    "X_train_vectorized2.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## n-grams\n",
    "One way to prevent misclassification is to add n-grams. \n",
    "For example, bigrams (2-grams) count pairs of adjacent words, and could give us features such as 'bad' versus 'not bad'.   \n",
    "Thus, we are refitting our training set (specifying a minimum document frequency), and extracting 1-grams and 2-grams."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28637\n"
     ]
    }
   ],
   "source": [
    "vect3 = CountVectorizer(min_df = min_df, ngram_range = (1,2)).fit(X_train)\n",
    "feat3 = vect3.get_feature_names()\n",
    "num_of_feat3 = len(feat3)\n",
    "print(num_of_feat3)\n",
    "\n",
    "X_train_vectorized3 = vect3.transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification Algorithms\n",
    "The following algorithms were used:  \n",
    "KNN, Logistic regression, Naive Bayes, Support Vector Machine (SVM), Feedforward neural network (1, 3 and 5 layers), LSTM, Bidirectional LSTM, GRU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-nearest Neighbour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "def KNN_classifier(X_train, y_train, X_test, y_test, vect):  \n",
    "    neigh = KNeighborsClassifier(n_neighbors=15)\n",
    "    neigh.fit(X_train, y_train)\n",
    "    neigh_pred = neigh.predict(vect.transform(X_test))\n",
    "    neigh_acc = accuracy_score(y_test, neigh_pred)\n",
    "    return neigh, neigh_pred, neigh_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def logistic_regr(X_train, y_train, X_test, y_test, vect):\n",
    "    # make an instance of the logistic regression curve\n",
    "    model = LogisticRegression(max_iter = 1000) # max_iter prevents it from crashing, as logistic regression is computationally heavy\n",
    "    \n",
    "    # fit the train data on the logistic model\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    # predict by inserting test data to the model\n",
    "    predictions = model.predict(vect.transform(X_test))\n",
    "    \n",
    "    # test the accuracy of the predictions\n",
    "    acc = accuracy_score(y_test, predictions)\n",
    "    \n",
    "    return model, predictions, acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def naive_bayes_classifier(X_train, y_train, X_test, y_test, vect):\n",
    "    bayes = naive_bayes.MultinomialNB()\n",
    "    bayes.fit(X_train, y_train)\n",
    "    bayes_pred = bayes.predict(vect.transform(X_test))\n",
    "    bayes_acc = accuracy_score(y_test, bayes_pred)\n",
    "    return bayes, bayes_pred, bayes_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support vector machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVM classifies data by separating the different classes with the help of a hyperplane between the classes.\n",
    "# The optimal hyperplane should be such as to maximize its distance as much as possible from all the data points from \n",
    "# any of the classes\n",
    "\n",
    "def SVM_classifier(X_train, y_train, X_test, y_test, vect):  \n",
    "    sv_model = svm.SVC()\n",
    "    sv_model.fit(X_train, y_train)\n",
    "    sv_pred = sv_model.predict(vect.transform(X_test))\n",
    "    sv_acc = accuracy_score(y_test, sv_pred)\n",
    "    return sv_model, sv_pred, sv_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple feedforward neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feedforward neural network with one hidden layer\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "\n",
    "def neural_net(X_train, y_train, X_test, y_test, vect):\n",
    "    n_epochs = 5\n",
    "    \n",
    "    # the size of the train test defines the number of input neurons\n",
    "    input_size = X_train.shape[1]\n",
    "    \n",
    "    model = tf.keras.Sequential()\n",
    "    \n",
    "    # hidden layer of 32 neurons\n",
    "    model.add(layers.Dense(16, input_dim = input_size))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # output layer of 1 neuron\n",
    "    model.add(layers.Dense(1, activation='sigmoid'))\n",
    "    model.summary()\n",
    "    \n",
    "    model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(0.01), metrics=['accuracy'])\n",
    "\n",
    "    model.fit(X_train, y_train, epochs=n_epochs)\n",
    "    \n",
    "    # predict the labels on test dataset\n",
    "    nn_pred = model.predict(vect.transform(X_test))\n",
    "    nn_pred = nn_pred.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate gives the loss and the accuracy of the model\n",
    "    results = model.evaluate(vect.transform(X_test), y_test)\n",
    "    \n",
    "    return model, nn_pred, results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-layer neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feedforward neural network with one hidden layer\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "\n",
    "def neural_deep_net(X_train, y_train, X_test, y_test, vect):\n",
    "    n_epochs = 5\n",
    "    \n",
    "    # the size of the train test defines the number of input neurons\n",
    "    input_size = X_train.shape[1]\n",
    "    \n",
    "    model = tf.keras.Sequential()\n",
    "    \n",
    "    # hidden layer 1\n",
    "    model.add(layers.Dense(16, input_dim = input_size))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # hidden layer 2\n",
    "    model.add(layers.Dense(32))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # hidden layer 3\n",
    "    model.add(layers.Dense(16))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # output layer of 1 neuron\n",
    "    model.add(layers.Dense(1, activation='sigmoid'))\n",
    "    model.summary()\n",
    "    \n",
    "    model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(0.01), metrics=['accuracy'])\n",
    "\n",
    "    model.fit(X_train, y_train, epochs=n_epochs)\n",
    "    \n",
    "    # predict the labels on test dataset\n",
    "    nn_pred = model.predict(vect.transform(X_test))\n",
    "    nn_pred = nn_pred.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate gives the loss and the accuracy of the model\n",
    "    results = model.evaluate(vect.transform(X_test), y_test)\n",
    "    \n",
    "    return model, nn_pred, results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-layer neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feedforward neural network with one hidden layer\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "\n",
    "def neural_deep_net2(X_train, y_train, X_test, y_test, vect):\n",
    "    n_epochs = 5\n",
    "    \n",
    "    # the size of the train test defines the number of input neurons\n",
    "    input_size = X_train.shape[1]\n",
    "    \n",
    "    model = tf.keras.Sequential()\n",
    "    \n",
    "    # hidden layer 1\n",
    "    model.add(layers.Dense(16, input_dim = input_size))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # hidden layer 2\n",
    "    model.add(layers.Dense(32))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    #hidden layer 3\n",
    "    model.add(layers.Dense(32))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    #hidden layer 4\n",
    "    model.add(layers.Dense(32))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # hidden layer 5\n",
    "    model.add(layers.Dense(16))\n",
    "    model.add(layers.Activation('relu'))\n",
    "    \n",
    "    # output layer of 1 neuron\n",
    "    model.add(layers.Dense(1, activation='sigmoid'))\n",
    "    model.summary()\n",
    "    \n",
    "    model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(0.01), metrics=['accuracy'])\n",
    "\n",
    "    model.fit(X_train, y_train, epochs=n_epochs)\n",
    "    \n",
    "    # predict the labels on test dataset\n",
    "    nn_pred = model.predict(vect.transform(X_test))\n",
    "    nn_pred = nn_pred.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate gives the loss and the accuracy of the model\n",
    "    results = model.evaluate(vect.transform(X_test), y_test)\n",
    "    \n",
    "    return model, nn_pred, results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running the algorithms of KNN, Logistic regression, Naive Bayes, Support Vector Machine (SVM), Feedforward neural network (1, 3 and 5 layers) on Bag of Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for Bag of Words\n",
      "KNN accuracy: 0.7725\n"
     ]
    }
   ],
   "source": [
    "# Bag of words\n",
    "print('Results for Bag of Words')\n",
    "neigh2, neigh_pred2, neigh_acc2 = KNN_classifier(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('KNN accuracy:', neigh_acc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression accuracy: 0.8615\n"
     ]
    }
   ],
   "source": [
    "model1, predictions1, acc1 = logistic_regr(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('Logistic regression accuracy:', acc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes accuracy: 0.8315\n"
     ]
    }
   ],
   "source": [
    "bayes1, bayes_pred1, bayes_acc1 = naive_bayes_classifier(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('Naive Bayes accuracy:', bayes_acc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Support vector machine accuracy: 0.845\n"
     ]
    }
   ],
   "source": [
    "sv_model1, sv_pred1, sv_acc1 = SVM_classifier(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('Support vector machine accuracy:', sv_acc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 16)                448512    \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 448,529\n",
      "Trainable params: 448,529\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 7s 923us/sample - loss: 0.3875 - accuracy: 0.8338\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 5s 641us/sample - loss: 0.0988 - accuracy: 0.9625\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 5s 590us/sample - loss: 0.0313 - accuracy: 0.9876\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 5s 581us/sample - loss: 0.0106 - accuracy: 0.9965\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 5s 584us/sample - loss: 0.0088 - accuracy: 0.9973\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 1s 518us/sample - loss: 0.7125 - accuracy: 0.8545\n",
      "\n",
      "Feedforward neural network accuracy: 0.8545\n"
     ]
    }
   ],
   "source": [
    "nn_classifier1, nn_pred1, result1 = neural_net(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('\\nFeedforward neural network accuracy:', result1[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 16)                448512    \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 449,601\n",
      "Trainable params: 449,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 5s 676us/sample - loss: 0.3945 - accuracy: 0.8263\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 5s 676us/sample - loss: 0.1232 - accuracy: 0.9549\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 5s 628us/sample - loss: 0.0520 - accuracy: 0.9804\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 5s 583us/sample - loss: 0.0398 - accuracy: 0.9846\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 5s 581us/sample - loss: 0.0269 - accuracy: 0.9891\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 1s 527us/sample - loss: 0.8147 - accuracy: 0.8510\n",
      "\n",
      "3-layer neural network accuracy: 0.851\n"
     ]
    }
   ],
   "source": [
    "deepmodel1, deep_pred1, deep_results1 = neural_deep_net(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('\\n3-layer neural network accuracy:', deep_results1[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 16)                448512    \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 451,713\n",
      "Trainable params: 451,713\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 5s 679us/sample - loss: 0.4465 - accuracy: 0.7980\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 5s 669us/sample - loss: 0.1574 - accuracy: 0.9399\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 5s 626us/sample - loss: 0.0767 - accuracy: 0.9703\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 5s 593us/sample - loss: 0.0596 - accuracy: 0.9741\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 5s 580us/sample - loss: 0.0474 - accuracy: 0.9811 - loss: 0.046\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 1s 520us/sample - loss: 0.7204 - accuracy: 0.8365 - loss: 0.7301 - accuracy\n",
      "\n",
      "5-layer neural network accuracy: 0.8365\n"
     ]
    }
   ],
   "source": [
    "deepmodel11, deep_pred11, deep_results11 = neural_deep_net2(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('\\n5-layer neural network accuracy:', deep_results11[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can observe that the results of the Machine Learning algorithms and the simple neural networks were very satisfying.   \n",
    "The accuracy in most cases was more than 80%:  \n",
    "KNN 77.25%  \n",
    "Logistic regression 86.15%  \n",
    "Naive Bayes 83.15%  \n",
    "Support vector machine 84.5%  \n",
    "Feedforward neural network 85.45%  \n",
    "3-layer neural network 83.85%  \n",
    "5-layer neural network 84.3%  \n",
    "\n",
    "The best performance for bag of words was given from Logistic regression: 86.15%."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running the algorithms of KNN, Logistic regression, Naive Bayes, Support Vector Machine (SVM), Feedforward neural network (1, 3 and 5 layers) on tf - idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for tf - idf\n",
      "KNN accuracy: 0.7725\n"
     ]
    }
   ],
   "source": [
    "# tf - idf\n",
    "print('Results for tf - idf')\n",
    "neigh2, neigh_pred2, neigh_acc2 = KNN_classifier(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('KNN accuracy:', neigh_acc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression accuracy: 0.8665\n"
     ]
    }
   ],
   "source": [
    "model2, predictions2, acc2 = logistic_regr(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('Logistic regression accuracy:', acc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes accuracy: 0.833\n"
     ]
    }
   ],
   "source": [
    "bayes2, bayes_pred2, bayes_acc2 = naive_bayes_classifier(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('Naive Bayes accuracy:', bayes_acc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Support vector machine accuracy: 0.877\n"
     ]
    }
   ],
   "source": [
    "sv_model2, sv_pred2, sv_acc2 = SVM_classifier(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('Support vector machine accuracy:', sv_acc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 16)                129488    \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 129,505\n",
      "Trainable params: 129,505\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 2s 224us/sample - loss: 0.3841 - accuracy: 0.8340\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 2s 224us/sample - loss: 0.1315 - accuracy: 0.9564\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 2s 197us/sample - loss: 0.0507 - accuracy: 0.9865\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 2s 200us/sample - loss: 0.0188 - accuracy: 0.9951\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 2s 200us/sample - loss: 0.0065 - accuracy: 0.9977\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 0s 161us/sample - loss: 0.6201 - accuracy: 0.8415\n",
      "\n",
      "Feedforward neural network accuracy: 0.8415\n"
     ]
    }
   ],
   "source": [
    "nn_classifier2, nn_pred2, result2 = neural_net(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('\\nFeedforward neural network accuracy:', result2[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_14 (Dense)             (None, 16)                129488    \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 130,577\n",
      "Trainable params: 130,577\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 2s 230us/sample - loss: 0.3961 - accuracy: 0.8189\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 2s 217us/sample - loss: 0.1426 - accuracy: 0.9524\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 2s 211us/sample - loss: 0.0692 - accuracy: 0.9755\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 2s 217us/sample - loss: 0.0378 - accuracy: 0.9850\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 2s 208us/sample - loss: 0.0251 - accuracy: 0.9893\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 0s 163us/sample - loss: 0.7777 - accuracy: 0.8320\n",
      "\n",
      "3-layer neural network accuracy: 0.832\n"
     ]
    }
   ],
   "source": [
    "deepmodel2, deep_pred2, deep_results2 = neural_deep_net(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('\\n3-layer neural network accuracy:', deep_results2[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_18 (Dense)             (None, 16)                129488    \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 132,689\n",
      "Trainable params: 132,689\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 2s 249us/sample - loss: 0.4106 - accuracy: 0.8073\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 2s 225us/sample - loss: 0.1483 - accuracy: 0.9465\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 2s 207us/sample - loss: 0.0689 - accuracy: 0.9743\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 2s 204us/sample - loss: 0.0295 - accuracy: 0.9868\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 2s 210us/sample - loss: 0.0226 - accuracy: 0.9911\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 0s 160us/sample - loss: 0.7296 - accuracy: 0.8445\n",
      "\n",
      "5-layer neural network accuracy: 0.8445\n"
     ]
    }
   ],
   "source": [
    "deepmodel22, deep_pred22, deep_results22 = neural_deep_net2(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('\\n5-layer neural network accuracy:', deep_results22[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy in most cases was again more than 80%:   \n",
    "KNN 77.25%  \n",
    "Logistic regression 86.65%  \n",
    "Naive Bayes 83.3%   \n",
    "Support vector machine 87.7%  \n",
    "Feedforward neural network 84.9%  \n",
    "3-layer neural network 83.9%  \n",
    "5-layer neural network 83.65%\n",
    "\n",
    "The best performance for tf - idf was given from support vector machine: 87.7%."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running the algorithms of KNN, Logistic regression, Naive Bayes, Support Vector Machine (SVM), Feedforward neural network (1, 3 and 5 layers) on n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for n-grams\n",
      "KNN accuracy: 0.6045\n"
     ]
    }
   ],
   "source": [
    "# ngrams\n",
    "print('Results for n-grams')\n",
    "neigh3, neigh_pred3, neigh_acc3 = KNN_classifier(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('KNN accuracy:', neigh_acc3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.879\n"
     ]
    }
   ],
   "source": [
    "model3, predictions3, acc3 = logistic_regr(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('Accuracy:', acc3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.871\n"
     ]
    }
   ],
   "source": [
    "bayes3, bayes_pred3, bayes_acc3 = naive_bayes_classifier(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('Accuracy:', bayes_acc3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.855\n"
     ]
    }
   ],
   "source": [
    "sv_model3, sv_pred3, sv_acc3 = SVM_classifier(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('Accuracy:', sv_acc3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_24 (Dense)             (None, 16)                458208    \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_25 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 458,225\n",
      "Trainable params: 458,225\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 6s 754us/sample - loss: 0.3580 - accuracy: 0.8470\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 5s 609us/sample - loss: 0.0554 - accuracy: 0.9809\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 5s 569us/sample - loss: 0.0083 - accuracy: 0.9965\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 5s 601us/sample - loss: 0.0031 - accuracy: 0.9979\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 5s 593us/sample - loss: 0.0023 - accuracy: 0.9984\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 1s 517us/sample - loss: 0.5814 - accuracy: 0.8795\n",
      "\n",
      "Accuracy: 0.8795\n"
     ]
    }
   ],
   "source": [
    "nn_classifier3, nn_pred3, result3 = neural_net(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('\\nAccuracy:', result3[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_26 (Dense)             (None, 16)                458208    \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 459,297\n",
      "Trainable params: 459,297\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 6s 724us/sample - loss: 0.3763 - accuracy: 0.8329 - loss: 0.3794 - accura\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 5s 659us/sample - loss: 0.0803 - accuracy: 0.9731 - loss: 0.0699 - \n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 5s 611us/sample - loss: 0.0199 - accuracy: 0.9927\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 5s 681us/sample - loss: 0.0154 - accuracy: 0.9934\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 5s 628us/sample - loss: 0.0222 - accuracy: 0.9920 - loss: 0.0225 - accu\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 1s 543us/sample - loss: 0.8097 - accuracy: 0.8550\n",
      "\n",
      "3-layer neural network accuracy: 0.855\n"
     ]
    }
   ],
   "source": [
    "deepmodel3, deep_pred3, deep_results3 = neural_deep_net(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('\\n3-layer neural network accuracy:', deep_results3[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_30 (Dense)             (None, 16)                458208    \n",
      "_________________________________________________________________\n",
      "activation_22 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 32)                544       \n",
      "_________________________________________________________________\n",
      "activation_23 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "activation_24 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_33 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "activation_25 (Activation)   (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_34 (Dense)             (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "activation_26 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 461,409\n",
      "Trainable params: 461,409\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 8000 samples\n",
      "Epoch 1/5\n",
      "8000/8000 [==============================] - 7s 888us/sample - loss: 0.3795 - accuracy: 0.8386\n",
      "Epoch 2/5\n",
      "8000/8000 [==============================] - 5s 664us/sample - loss: 0.0771 - accuracy: 0.9729\n",
      "Epoch 3/5\n",
      "8000/8000 [==============================] - 5s 634us/sample - loss: 0.0222 - accuracy: 0.9926\n",
      "Epoch 4/5\n",
      "8000/8000 [==============================] - 5s 637us/sample - loss: 0.0256 - accuracy: 0.9908 - loss: 0.0256 - accuracy: 0.99\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 5s 634us/sample - loss: 0.0228 - accuracy: 0.9916\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "2000/2000 [==============================] - 1s 535us/sample - loss: 0.7388 - accuracy: 0.8690\n",
      "\n",
      "5-layer neural network accuracy: 0.869\n"
     ]
    }
   ],
   "source": [
    "deepmodel33, deep_pred33, deep_results33 = neural_deep_net2(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('\\n5-layer neural network accuracy:', deep_results33[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy in most cases was again more than 80%:   \n",
    "KNN 60.45%  \n",
    "Logistic regression 87.9%  \n",
    "Naive Bayes 87.1%  \n",
    "Support vector machine 85.5%  \n",
    "Feedforward neural network 87%    \n",
    "3-layer neural network 87.6%  \n",
    "5-layer neural network 86.2%\n",
    "\n",
    "The best performance for n-grams was given from logistic regression: 87.9%."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see the most frequently used negative and positive words according to each model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def frequent_words(features, model):\n",
    "    feature_names = np.array(features)\n",
    "    # sort based on positive or negative weight for each word\n",
    "    # smaller coefficient - more negative\n",
    "    # larger coefficient - more positive\n",
    "    sorted_coef_index = model.coef_[0].argsort()\n",
    "    print('Smallest Coefs: \\n{}\\n'.format(feature_names[sorted_coef_index[:10]]))\n",
    "    print('Largest Coefs: \\n{}\\n'.format(feature_names[sorted_coef_index[:-11:-1]]))\n",
    "    return "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bag of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression most frequent words on bag of words\n",
      "Smallest Coefs: \n",
      "['boring' 'poor' 'worst' 'waste' 'disappointing' 'disappointed'\n",
      " 'disappointment' 'stopped' 'horrible' 'terrible']\n",
      "\n",
      "Largest Coefs: \n",
      "['excellent' 'perfect' 'loves' 'fantastic' 'wonderful' 'amazing' 'awesome'\n",
      " 'great' 'extra' 'pleased']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Here we can see the most frequent negative and positive words for each model \n",
    "print('Logistic regression most frequent words on bag of words')\n",
    "frequent_words(features, model1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf - idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression most frequent words on tf - idf\n",
      "Smallest Coefs: \n",
      "['not' 'boring' 'waste' 'worst' 'poor' 'bad' 'disappointed' 'money' 'don'\n",
      " 'disappointing']\n",
      "\n",
      "Largest Coefs: \n",
      "['great' 'excellent' 'love' 'best' 'good' 'perfect' 'well' 'and' 'easy'\n",
      " 'wonderful']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Logistic regression most frequent words on tf - idf')\n",
    "frequent_words(feat2, model2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n - grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression most frequent words on n-grams\n",
      "Smallest Coefs: \n",
      "['boring' 'poor' 'disappointed' 'disappointing' 'worst' 'waste'\n",
      " 'the worst' 'not good' 'not worth' 'disappointment']\n",
      "\n",
      "Largest Coefs: \n",
      "['excellent' 'perfect' 'great' 'love' 'amazing' 'loves' 'better than'\n",
      " 'awesome' 'wonderful' 'not bad']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Logistic regression most frequent words on n-grams')\n",
    "frequent_words(feat3, model3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM & GRU with Embeddings results\n",
    "We add them separately, as they take more time to run than the rest of the methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This implementation is very slow to train\n",
    "\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras.layers import LSTM\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "\n",
    "def LSTM_net(X_train, y_train, X_test, y_test, vect):\n",
    "    input_size = X_train.shape[1]\n",
    "    embed_dim = 10\n",
    "    lstm_dim = 10\n",
    "    batch_size = 32\n",
    "    n_epochs = 3\n",
    "\n",
    "    model = Sequential()\n",
    "    \n",
    "    # Add the word embedding Layer\n",
    "    model.add(Embedding(10, embed_dim, input_length = input_size))\n",
    "    \n",
    "    # Add the LSTM Layer\n",
    "    model.add(LSTM(lstm_dim))\n",
    "    \n",
    "    # Output layer\n",
    "    model.add(Dense(1,activation='softmax'))\n",
    "    \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "    model.fit(X_train, y_train, epochs = n_epochs)\n",
    "    \n",
    "    # predict the labels on test dataset\n",
    "    lstm_pred = model.predict(vect.transform(X_test))\n",
    "    lstm_pred = nn_pred.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate gives the loss and the accuracy of the model\n",
    "    lstm_results = model.evaluate(vect.transform(X_test), y_test)\n",
    "    \n",
    "    return model, lstm_pred, lstm_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bidirectional LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This implementation is very slow to train\n",
    "\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras.layers import LSTM\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Bidirectional\n",
    "\n",
    "def BiLSTM_net(X_train, y_train, X_test, y_test, vect):\n",
    "    input_size = X_train.shape[1]\n",
    "    embed_dim = 10\n",
    "    lstm_dim = 10\n",
    "    batch_size = 32\n",
    "    n_epochs = 3\n",
    "\n",
    "    model = Sequential()\n",
    "    \n",
    "    # Add the word embedding Layer\n",
    "    model.add(Embedding(1000, embed_dim, input_length = input_size))\n",
    "    \n",
    "    # Add the LSTM Layer\n",
    "    model.add(Bidirectional(LSTM(lstm_dim)))\n",
    "    \n",
    "    # Output layer\n",
    "    model.add(Dense(1,activation='softmax'))\n",
    "    \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "    model.fit(X_train, y_train, epochs = n_epochs)\n",
    "    \n",
    "    # predict the labels on test dataset\n",
    "    BiLSTM_pred = model.predict(vect.transform(X_test))\n",
    "    BiLSTM_pred = nn_pred.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate gives the loss and the accuracy of the model\n",
    "    BiLSTM_results = model.evaluate(vect.transform(X_test), y_test)\n",
    "    \n",
    "    return model, BiLSTM_pred, BiLSTM_results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This implementation is very slow to train\n",
    "\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras.layers import GRU\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "\n",
    "def GRU_net(X_train, y_train, X_test, y_test, vect):\n",
    "    input_size = X_train.shape[1]\n",
    "    embed_dim = 10\n",
    "    gru_dim = 10\n",
    "    batch_size = 32\n",
    "    n_epochs = 3\n",
    "\n",
    "    model = Sequential()\n",
    "    \n",
    "    # Add the word embedding Layer\n",
    "    model.add(Embedding(1000, embed_dim, input_length = input_size))\n",
    "    \n",
    "    # Add the LSTM Layer\n",
    "    model.add(GRU(gru_dim))\n",
    "    \n",
    "    # Output layer\n",
    "    model.add(Dense(1,activation='softmax'))\n",
    "    \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "    model.fit(X_train, y_train, epochs = n_epochs)\n",
    "    \n",
    "    # predict the labels on test dataset\n",
    "    gru_pred = model.predict(vect.transform(X_test))\n",
    "    gru_pred = nn_pred.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate gives the loss and the accuracy of the model\n",
    "    gru_results = model.evaluate(vect.transform(X_test), y_test)\n",
    "    \n",
    "    return model, gru_pred, gru_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM on bag of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_model1, lstm_pred1, lstm_results1 = LSTM_net(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('LSTM for bag of words')\n",
    "print('Accuracy:', lstm_results1[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM on Tf-idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_model2, lstm_pred2, lstm_results2 = LSTM_net(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('LSTM for Tf-idf')\n",
    "print('Accuracy:', lstm_results2[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM on bag of n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_model3, lstm_pred3, lstm_results3 = LSTM_net(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('LSTM for n-grams')\n",
    "print('Accuracy:', lstm_results3[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BiLSTM on bag of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BiLSTM_model1, BiLSTM_pred1, BiLSTM_results1 = BiLSTM_net(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('Bidirectional LSTM for bag of words')\n",
    "print('Accuracy:', BiLSTM_results1[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BiLSTM on Tf-idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BiLSTM_model2, BiLSTM_pred2, BiLSTM_results2 = BiLSTM_net(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('Bidirectional LSTM for Tf-idf')\n",
    "print('Accuracy:', BiLSTM_results2[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BiLSTM on n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BiLSTM_model3, BiLSTM_pred3, BiLSTM_results3 = BiLSTM_net(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('Bidirectional LSTM for n-grams')\n",
    "print('Accuracy:', BiLSTM_results3[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GRU on bag of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_model1, gru_pred1, gru_results1 = GRU_net(X_train_vectorized1, y_train, X_test, y_test, vect1)\n",
    "print('GRU for bag of words')\n",
    "print('Accuracy:', gru_results1[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GRU on bag of Tf-idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_model2, gru_pred2, gru_results2 = GRU_net(X_train_vectorized2, y_train, X_test, y_test, vect2)\n",
    "print('GRU for Tf-idf')\n",
    "print('Accuracy:', gru_results2[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GRU on bag of n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_model3, gru_pred3, gru_results3 = GRU_net(X_train_vectorized3, y_train, X_test, y_test, vect3)\n",
    "print('GRU for n-grams')\n",
    "print('Accuracy:', gru_results3[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}