GRU+GloVe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys, os, re, csv, codecs, numpy as np, pandas as pd\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, GRU, Embedding, Dropout, Activation\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../input/'\n",
    "EMBEDDING_FILE=f'{path}glove.6B.50d.txt'\n",
    "TRAIN_DATA_FILE=f'{path}train.csv'\n",
    "TEST_DATA_FILE=f'{path}test.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set some basic config parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 50 # how big is each word vector\n",
    "max_features = 20000 # how many unique words to use (i.e num rows in embedding vector)\n",
    "maxlen = 100 # max number of words in a comment to use"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in our data and replace missing values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(TRAIN_DATA_FILE)\n",
    "test = pd.read_csv(TEST_DATA_FILE)\n",
    "\n",
    "list_sentences_train = train[\"comment_text\"].fillna(\"_na_\").values\n",
    "list_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "y = train[list_classes].values\n",
    "list_sentences_test = test[\"comment_text\"].fillna(\"_na_\").values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Standard keras preprocessing, to turn each comment into a list of word indexes of equal length (with truncation or padding as needed)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=max_features)\n",
    "tokenizer.fit_on_texts(list(list_sentences_train))\n",
    "list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)\n",
    "list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)\n",
    "X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)\n",
    "X_te = pad_sequences(list_tokenized_test, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the glove word vectors (space delimited strings) into a dictionary from word->vector."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "embeddings_index = dict(get_coefs(*o.strip().split()) for o in open(EMBEDDING_FILE))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use these vectors to create our embedding matrix, with random initialization for words that aren't in GloVe. We'll use the same mean and stdev of embeddings the GloVe has when generating the random init."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.020940498, 0.6441043)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_embs = np.stack(embeddings_index.values())\n",
    "emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "emb_mean,emb_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index\n",
    "nb_words = min(max_features, len(word_index))\n",
    "embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple bidirectional GRU with two fully connected layers. We add some dropout to the LSTM since even 2 epochs is enough to overfit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = Input(shape=(maxlen,))\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "x = Bidirectional(GRU(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "x = GlobalMaxPool1D()(x)\n",
    "x = Dense(50, activation=\"relu\")(x)\n",
    "x = Dropout(0.1)(x)\n",
    "x = Dense(6, activation=\"sigmoid\")(x)\n",
    "model = Model(inputs=inp, outputs=x)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we're ready to fit out model! Use validation_split when not submitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 143613 samples, validate on 15958 samples\n",
      "Epoch 1/2\n",
      "143613/143613 [==============================] - 687s 5ms/step - loss: 0.0582 - acc: 0.9797 - val_loss: 0.0482 - val_acc: 0.9825\n",
      "Epoch 2/2\n",
      "143613/143613 [==============================] - 702s 5ms/step - loss: 0.0443 - acc: 0.9832 - val_loss: 0.0457 - val_acc: 0.9829\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_t, y, batch_size=32, epochs=2, validation_split=0.1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "153164/153164 [==============================] - 141s 919us/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict([X_te], batch_size=1024, verbose=1)\n",
    "sample_submission = pd.read_csv(f'{path}sample_submission.csv')\n",
    "sample_submission[list_classes] = y_test\n",
    "sample_submission.to_csv('submission_gru.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}