CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from gensim import models\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Dense, LSTM, GRU, Dropout, Reshape, Flatten, concatenate, Input, MaxPooling1D, Conv1D,GlobalAveragePooling1D, GlobalMaxPooling1D, Embedding\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('train.csv')\n",
    "data.dropna(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "data['tokens'] = data['TEXT'].apply(lambda x: tokenizer.tokenize(x))\n",
    "\n",
    "pos = []\n",
    "neg = []\n",
    "for l in data.truth:\n",
    "    if l == 0:\n",
    "        pos.append(0)\n",
    "        neg.append(1)\n",
    "    elif l == 1:\n",
    "        pos.append(1)\n",
    "        neg.append(0)\n",
    "        \n",
    "data['Pos']= pos\n",
    "data['Neg']= neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>tokens</th>\n",
       "      <th>truth</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hahaha wanted say comment made laugh</td>\n",
       "      <td>[hahaha, wanted, say, comment, made, laugh]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>feel ya</td>\n",
       "      <td>[feel, ya]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kind superior power trajectory definitely laun...</td>\n",
       "      <td>[kind, superior, power, trajectory, definitely...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>delete script tampermonkey option puushvqwgafe...</td>\n",
       "      <td>[delete, script, tampermonkey, option, puushvq...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>end sequel trilogy</td>\n",
       "      <td>[end, sequel, trilogy]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                TEXT  \\\n",
       "0               hahaha wanted say comment made laugh   \n",
       "1                                            feel ya   \n",
       "2  kind superior power trajectory definitely laun...   \n",
       "3  delete script tampermonkey option puushvqwgafe...   \n",
       "4                                 end sequel trilogy   \n",
       "\n",
       "                                              tokens  truth  Pos  Neg  \n",
       "0        [hahaha, wanted, say, comment, made, laugh]      0    0    1  \n",
       "1                                         [feel, ya]      0    0    1  \n",
       "2  [kind, superior, power, trajectory, definitely...      0    0    1  \n",
       "3  [delete, script, tampermonkey, option, puushvq...      0    0    1  \n",
       "4                             [end, sequel, trilogy]      0    0    1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data[['TEXT', 'tokens', 'truth', 'Pos', 'Neg']]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train, data_test = train_test_split(data, \n",
    "                                         test_size=0.001, \n",
    "                                         random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2014356 words total, with a vocabulary size of 99367\n",
      "Max sentence length is 3534\n"
     ]
    }
   ],
   "source": [
    "all_training_words = [word for tokens in data_train[\"tokens\"] for word in tokens]\n",
    "training_sentence_lengths = [len(tokens) for tokens in data_train[\"tokens\"]]\n",
    "TRAINING_VOCAB = sorted(list(set(all_training_words)))\n",
    "\n",
    "print(\"%s words total, with a vocabulary size of %s\" % (len(all_training_words), len(TRAINING_VOCAB)))\n",
    "print(\"Max sentence length is %s\" % max(training_sentence_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2174 words total, with a vocabulary size of 1301\n",
      "Max sentence length is 220\n"
     ]
    }
   ],
   "source": [
    "all_test_words = [word for tokens in data_test[\"tokens\"] for word in tokens]\n",
    "test_sentence_lengths = [len(tokens) for tokens in data_test[\"tokens\"]]\n",
    "TEST_VOCAB = sorted(list(set(all_test_words)))\n",
    "print(\"%s words total, with a vocabulary size of %s\" % (len(all_test_words), len(TEST_VOCAB)))\n",
    "print(\"Max sentence length is %s\" % max(test_sentence_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "\n",
    "model = gensim.models.Word2Vec(sentences = data['tokens'], size=EMBEDDING_DIM, window=5, workers=4, min_count=1)\n",
    "filename = 'gensim_word2vec_model.txt'\n",
    "model.wv.save_word2vec_format(filename, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_path = 'GoogleNews-vectors-negative300.bin.gz'\n",
    "word2vec = models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_average_word2vec(tokens_list, vector, generate_missing=False, k=300):\n",
    "    if len(tokens_list)<1:\n",
    "        return np.zeros(k)\n",
    "    if generate_missing:\n",
    "        vectorized = [vector[word] if word in vector else np.random.rand(k) for word in tokens_list]\n",
    "    else:\n",
    "        vectorized = [vector[word] if word in vector else np.zeros(k) for word in tokens_list]\n",
    "    length = len(vectorized)\n",
    "    summed = np.sum(vectorized, axis=0)\n",
    "    averaged = np.divide(summed, length)\n",
    "    return averaged\n",
    "\n",
    "def get_word2vec_embeddings(vectors, clean_comments, generate_missing=False):\n",
    "    embeddings = clean_comments['tokens'].apply(lambda x: get_average_word2vec(x, vectors, \n",
    "                                                                                generate_missing=generate_missing))\n",
    "    return list(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_embeddings = get_word2vec_embeddings(word2vec, data_train, generate_missing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 10000\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 99367 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words=len(TRAINING_VOCAB), lower=True, char_level=False)\n",
    "tokenizer.fit_on_texts(data_train[\"TEXT\"].tolist())\n",
    "training_sequences = tokenizer.texts_to_sequences(data_train[\"TEXT\"].tolist())\n",
    "\n",
    "train_word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(train_word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(99368, 300)\n"
     ]
    }
   ],
   "source": [
    "train_cnn_data = pad_sequences(training_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "train_embedding_weights = np.zeros((len(train_word_index)+1, EMBEDDING_DIM))\n",
    "for word,index in train_word_index.items():\n",
    "    train_embedding_weights[index,:] = word2vec[word] if word in word2vec else np.random.rand(EMBEDDING_DIM)\n",
    "print(train_embedding_weights.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sequences = tokenizer.texts_to_sequences(data_test[\"TEXT\"].tolist())\n",
    "test_cnn_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ConvNet(embeddings, max_sequence_length, num_words, embedding_dim, labels_index):\n",
    "    \n",
    "    embedding_layer = Embedding(num_words,\n",
    "                            embedding_dim,\n",
    "                            weights=[embeddings],\n",
    "                            input_length=max_sequence_length,\n",
    "                            trainable=False)\n",
    "    \n",
    "    sequence_input = Input(shape=(max_sequence_length,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "    convs = []\n",
    "    filter_sizes = [2,3,4,5,6]\n",
    "\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Conv1D(filters=200, kernel_size=filter_size, activation='relu')(embedded_sequences)\n",
    "        l_pool = GlobalMaxPooling1D()(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "\n",
    "    l_merge = concatenate(convs, axis=1)\n",
    "\n",
    "    x = Dropout(0.1)(l_merge)  \n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    preds = Dense(labels_index, activation='sigmoid')(x)\n",
    "\n",
    "    model = Model(sequence_input, preds)\n",
    "    \n",
    "    \n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['acc'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = ['Pos', 'Neg']\n",
    "y_train = data_train[label_names].values\n",
    "x_train = train_cnn_data\n",
    "y_tr = y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 10000)        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 10000, 300)   29810400    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 9999, 200)    120200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 9998, 200)    180200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 9997, 200)    240200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 9996, 200)    300200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 9995, 200)    360200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 200)          0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 200)          0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 200)          0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_4 (GlobalM (None, 200)          0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_5 (GlobalM (None, 200)          0           conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 1000)         0           global_max_pooling1d_1[0][0]     \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "                                                                 global_max_pooling1d_4[0][0]     \n",
      "                                                                 global_max_pooling1d_5[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1000)         0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 128)          128128      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 128)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 2)            258         dropout_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 31,139,786\n",
      "Trainable params: 1,329,386\n",
      "Non-trainable params: 29,810,400\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = ConvNet(train_embedding_weights, MAX_SEQUENCE_LENGTH, len(train_word_index)+1, EMBEDDING_DIM, len(list(label_names)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 40\n",
    "batch_size = 32\n",
    "\n",
    "hist = model.fit(x_train, y_tr, epochs=num_epochs, validation_split=0.1, shuffle=True, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('CNN-new.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97     11938\n",
      "           1       0.42      0.12      0.19       685\n",
      "\n",
      "    accuracy                           0.94     12623\n",
      "   macro avg       0.69      0.56      0.58     12623\n",
      "weighted avg       0.92      0.94      0.93     12623\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "predictions = model.predict(test_cnn_data, batch_size=1024, verbose=1)\n",
    "\n",
    "temp = [1 if x==0 else 0 for x in y_pred_bool]\n",
    "print(classification_report(data_test['truth'], temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "1/1 [==============================] - 0s 17ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words=len(TRAINING_VOCAB), lower=True, char_level=False)\n",
    "tokenizer.fit_on_texts(data_train[\"TEXT\"].tolist())\n",
    "temprun = tokenizer.texts_to_sequences(['Darkness my old fren, ive come to talk to u agen'])\n",
    "temprun = pad_sequences(temprun, maxlen=200)\n",
    "\n",
    "predictions = model.predict(temprun, batch_size=1024, verbose=1)\n",
    "y_pred = np.argmax(predictions, axis=1)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    12484\n",
       "0      139\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = pd.DataFrame(y_pred_bool)\n",
    "data_test['truth'].value_counts()\n",
    "#temp[0].value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow1.1] *",
   "language": "python",
   "name": "conda-env-tensorflow1.1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}