train_char_level.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "import sotoxic.models.keras.model_zoo as model_zoo\n",
    "from sotoxic.train import trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = 'Dataset/'\n",
    "TRAIN_DATA_FILE=path + 'train.csv'\n",
    "TEST_DATA_FILE=path + 'test.csv'\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 1200\n",
    "EMBEDDING_DIM = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(TRAIN_DATA_FILE)\n",
    "test_df = pd.read_csv(TEST_DATA_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "# Load the cleaned words\n",
    "########################################\n",
    "\n",
    "cl_path = 'features/cleanwords.txt'\n",
    "clean_word_dict = {}\n",
    "with open(cl_path, 'r', encoding='utf-8') as cl:\n",
    "    for line in cl:\n",
    "        line = line.strip('\\n')\n",
    "        typo, correct = line.split(',')\n",
    "        clean_word_dict[typo] = correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## process texts in datasets\n",
    "########################################\n",
    "print('Processing text dataset')\n",
    "from collections import defaultdict\n",
    "# Regex to remove all Non-Alpha Numeric and space\n",
    "special_character_removal=re.compile(r'[^a-z\\d ]',re.IGNORECASE)\n",
    "\n",
    "# regex to replace all numerics\n",
    "replace_numbers=re.compile(r'\\d+',re.IGNORECASE)\n",
    "word_count_dict = defaultdict(int)\n",
    "import re\n",
    "toxic_dict = {}\n",
    "\n",
    "def clean_text(text, remove_stopwords=False, stem_words=False, count_null_words=True):\n",
    "    # Clean the text, with the option to remove stopwords and to stem words.\n",
    "    text = re.sub(r\"https?:\\/\\/(www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)\", \"\", text)\n",
    "    text = re.sub(r\"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}\", \"\", text)\n",
    "    \n",
    "    for typo, correct in clean_word_dict.items():\n",
    "        text = re.sub(typo, \" \" + correct + \" \", text)\n",
    "\n",
    "    text = re.sub(r\"what's\", \"what is \", text)\n",
    "    text = re.sub(r\"\\'s\", \" \", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text = re.sub(r\"can't\", \"cannot \", text)\n",
    "    text = re.sub(r\"n't\", \" not \", text)\n",
    "    text = re.sub(r\"i'm\", \"i am \", text)\n",
    "    text = re.sub(r\"i’m\", \"i am\", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\",\", \" \", text)\n",
    "    text = re.sub(r\"\\.\", \" \", text)\n",
    "    text = re.sub(r\"!\", \" ! \", text)\n",
    "    text = re.sub(r\"\\/\", \" \", text)\n",
    "    text = re.sub(r\"\\^\", \" ^ \", text)\n",
    "    text = re.sub(r\"\\+\", \" + \", text)\n",
    "    text = re.sub(r\"\\-\", \" - \", text)\n",
    "    text = re.sub(r\"\\=\", \" = \", text)\n",
    "    text = re.sub(r\"'\", \" \", text)\n",
    "    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n",
    "    text = re.sub(r\":\", \" : \", text)\n",
    "    text = re.sub(r\" e g \", \" eg \", text)\n",
    "    text = re.sub(r\" b g \", \" bg \", text)\n",
    "    text = re.sub(r\" u s \", \" american \", text)\n",
    "    text = re.sub(r\"\\0s\", \"0\", text)\n",
    "    text = re.sub(r\" 9 11 \", \"911\", text)\n",
    "    text = re.sub(r\"e - mail\", \"email\", text)\n",
    "    text = re.sub(r\"j k\", \"jk\", text)\n",
    "    text = re.sub(r\"\\s{2,}\", \" \", text)\n",
    "    text = replace_numbers.sub('', text)\n",
    "    #text = special_character_removal.sub('',text)\n",
    "\n",
    "    if count_null_words:\n",
    "        text = text.split()\n",
    "        for t in text:\n",
    "            word_count_dict[t] += 1\n",
    "        text = \" \".join(text)\n",
    "    \n",
    "    # Optionally, shorten words to their stems\n",
    "    if stem_words:\n",
    "        text = text.split()\n",
    "        stemmer = SnowballStemmer('english')\n",
    "        stemmed_words = [stemmer.stem(word) for word in text]\n",
    "        text = \" \".join(stemmed_words)\n",
    "\n",
    "    return (text)\n",
    "\n",
    "list_sentences_train = train_df[\"comment_text\"].fillna(\"no comment\").values\n",
    "list_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "y = train_df[list_classes].values\n",
    "list_sentences_test = test_df[\"comment_text\"].fillna(\"no comment\").values\n",
    "\n",
    "comments = [clean_text(text) for text in list_sentences_train]    \n",
    "test_comments=[clean_text(text) for text in list_sentences_test]\n",
    "\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS, char_level=True, lower=False)\n",
    "tokenizer.fit_on_texts(comments + test_comments)\n",
    "\n",
    "sequences = tokenizer.texts_to_sequences(comments)\n",
    "test_sequences = tokenizer.texts_to_sequences(test_comments)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "nb_words = len(word_index)\n",
    "print('Found %s unique tokens' % len(word_index))\n",
    "\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of data tensor:', data.shape)\n",
    "print('Shape of label tensor:', y.shape)\n",
    "\n",
    "test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of test_data tensor:', test_data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_charbased_model():\n",
    "    return model_zoo.get_av_rnn(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, out_size=7)\n",
    "\n",
    "keras_model_trainer = trainer.KerasModelTrainer(model_stamp='char_avrnn', epoch_num=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models, val_loss, total_auc, fold_predictions = keras_model_trainer.train_folds(data, y, fold_count=10, batch_size=512,\n",
    "                                                                                get_model_func=get_charbased_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Overall val-loss:\", val_loss, \"AUC\", total_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_fold_preditcions = np.concatenate(fold_predictions, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_auc = roc_auc_score(y[:-1], train_fold_preditcions)\n",
    "print(\"Training AUC\", training_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test_data = test_df\n",
    "CLASSES = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "submit_path_prefix = \"results/rnn/char/Char-avrnn-\" + str(MAX_NB_WORDS) + \"-RST-nlp-ct-\" + str(MAX_SEQUENCE_LENGTH) \n",
    "\n",
    "print(\"Predicting testing results...\")\n",
    "test_predicts_list = []\n",
    "for fold_id, model in enumerate(models):\n",
    "    test_predicts = model.predict(test_data, batch_size=256, verbose=1)\n",
    "    test_predicts_list.append(test_predicts[:, :-1])\n",
    "    np.save(\"predict_path/\", test_predicts)\n",
    "\n",
    "test_predicts = np.zeros(test_predicts_list[0].shape)\n",
    "for fold_predict in test_predicts_list:\n",
    "    test_predicts += fold_predict\n",
    "test_predicts /= len(test_predicts_list)\n",
    "\n",
    "test_ids = test_df[\"id\"].values\n",
    "test_ids = test_ids.reshape((len(test_ids), 1))\n",
    "\n",
    "test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)\n",
    "test_predicts[\"id\"] = test_ids\n",
    "test_predicts = test_predicts[[\"id\"] + CLASSES]\n",
    "submit_path = submit_path_prefix + \"-L{:4f}-A{:4f}.csv\".format(val_loss, total_auc)\n",
    "test_predicts.to_csv(submit_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Predicting training results...\")\n",
    "\n",
    "train_ids = train_df[\"id\"].values\n",
    "train_ids = train_ids.reshape((len(train_ids), 1))\n",
    "\n",
    "train_predicts = pd.DataFrame(data=train_fold_preditcions, columns=CLASSES)\n",
    "train_predicts[\"id\"] = train_ids\n",
    "train_predicts = train_predicts[[\"id\"] + CLASSES]\n",
    "submit_path = submit_path_prefix + \"-Train-L{:4f}-A{:4f}.csv\".format(val_loss, training_auc)\n",
    "train_predicts.to_csv(submit_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}