demo-MovieReviews-mturk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.cm as cm\n",
    "from matplotlib import pyplot as plt \n",
    "from keras.models import Model\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense, Input\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# packages for learning from crowds\n",
    "from crowd_layer.crowd_layers import CrowdsRegression, MaskedMultiMSE\n",
    "\n",
    "# prevent tensorflow from allocating the entire GPU memory at once\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_RUNS = 30\n",
    "DATA_PATH = \"/home/fmpr/datasets/deep-crowds-datasets/MovieReviews/\"\n",
    "BATCH_SIZE = 128\n",
    "N_EPOCHS = 100\n",
    "GLOVE_DIR = \"/home/fmpr/datasets/glove.6B/\"\n",
    "MAX_SEQUENCE_LENGTH = 1000\n",
    "MAX_NB_WORDS = 20000\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_texts(filename):\n",
    "    f = open(filename)\n",
    "    data = [line.strip() for line in f]\n",
    "    f.close()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num. train texts: 1498\n",
      "Num. test texts:  3508\n"
     ]
    }
   ],
   "source": [
    "texts_all = read_texts(DATA_PATH+\"texts_all.txt\")\n",
    "texts_train = read_texts(DATA_PATH+\"texts_train.txt\")\n",
    "targets_train = np.loadtxt(DATA_PATH+\"ratings_train.txt\")\n",
    "targets_train_mean = np.loadtxt(DATA_PATH+\"ratings_train_mean.txt\")\n",
    "targets_train_ds = np.loadtxt(DATA_PATH+\"ratings_train_DS.txt\")\n",
    "texts_test = read_texts(DATA_PATH+\"texts_test.txt\")\n",
    "targets_test = np.loadtxt(DATA_PATH+\"ratings_test.txt\")\n",
    "\n",
    "print(\"Num. train texts: %d\" % len(texts_train))\n",
    "print(\"Num. test texts:  %d\" % len(texts_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load crowdsourced answers from Mechanical Turk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AMT answers matrix shape: (1498, 135)\n",
      "Num. annotators: 135\n"
     ]
    }
   ],
   "source": [
    "answers = pd.read_csv(DATA_PATH+\"answers.txt\", header=None, delimiter=\" \").as_matrix()\n",
    "answers = answers[:,:-1]\n",
    "print(\"AMT answers matrix shape: %s\" % str(answers.shape))\n",
    "N_ANNOT = answers.shape[1]\n",
    "print(\"Num. annotators: %d\" % N_ANNOT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Standerdize targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean target: 0.574\n",
      "Std. target: 0.183\n"
     ]
    }
   ],
   "source": [
    "mean_target = np.mean(targets_train)\n",
    "std_target = np.std(targets_train)\n",
    "print(\"Mean target: %.3f\" % mean_target)\n",
    "print(\"Std. target: %.3f\" % std_target)\n",
    "targets_train = (np.array(targets_train) - mean_target) / std_target\n",
    "targets_train_mean = (np.array(targets_train_mean) - mean_target) / std_target\n",
    "targets_train_ds = (np.array(targets_train_ds) - mean_target) / std_target\n",
    "targets_test = (np.array(targets_test) - mean_target) / std_target\n",
    "for i in xrange(answers.shape[0]):\n",
    "    for r in xrange(answers.shape[1]):\n",
    "        if answers[i,r] != -1:\n",
    "            answers[i,r] = (answers[i,r] - mean_target) / std_target\n",
    "        else:\n",
    "        \tanswers[i,r] = 999999999 # use this ugly trick to encode a missing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build index mapping words in the embeddings set to their embedding vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeddings_index = {}\n",
    "f = open(GLOVE_DIR + 'glove.6B.300d.txt')\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize the text samples into a 2D integer tensor and pad sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 46259 unique tokens.\n",
      "('Shape of train tensor:', (1498, 1000))\n",
      "('Shape of test tensor:', (3508, 1000))\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts_all)\n",
    "sequences_train = tokenizer.texts_to_sequences(texts_train)\n",
    "sequences_test = tokenizer.texts_to_sequences(texts_test)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "data_train = pad_sequences(sequences_train, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_test = pad_sequences(sequences_test, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of train tensor:', data_train.shape)\n",
    "print('Shape of test tensor:', data_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing embedding matrix.\n"
     ]
    }
   ],
   "source": [
    "print('Preparing embedding matrix.')\n",
    "num_words = min(MAX_NB_WORDS, len(word_index))\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if i >= MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the base deep learning model\n",
    "\n",
    "Here we shall use features representation produced by the VGG16 network as the input. Our base model is then simply composed by one densely-connected layer with 128 hidden units and an output dense layer. We use 50% dropout between the two dense layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_base_model():\n",
    "    # load pre-trained word embeddings into an Embedding layer\n",
    "    # note that we set trainable = False so as to keep the embeddings fixed\n",
    "    embedding_layer = Embedding(num_words,\n",
    "                                EMBEDDING_DIM,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                trainable=False)\n",
    "\n",
    "    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "    x = Conv1D(128, 3, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(5)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Conv1D(128, 5, activation='relu')(x)\n",
    "    x = MaxPooling1D(5)(x)\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(32, activation='relu')(x)\n",
    "    preds = Dense(1, activation='linear')(x)\n",
    "\n",
    "    base_model = Model(sequence_input, preds)\n",
    "    base_model.compile(loss='mse', optimizer='adam')\n",
    "\n",
    "    return base_model, sequence_input, preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary function for evaluating the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_error(trues, predicted):\n",
    "    corr = np.corrcoef(predicted, trues)[0,1]\n",
    "    mae = np.mean(np.abs(predicted - trues))\n",
    "    mse = np.mean((predicted - trues)**2)\n",
    "    rmse = np.sqrt(np.mean((predicted - trues)**2))\n",
    "    r2 = max(0, 1 - np.sum((trues-predicted)**2) / np.sum((trues - np.mean(trues))**2))\n",
    "\n",
    "    return corr, mae, mse, rmse, r2\n",
    "\n",
    "def eval_model(model, test_data, test_labels):\n",
    "    # testset error\n",
    "    predicted = model.predict(test_data)[:,0] * std_target + mean_target\n",
    "    trues = test_labels * std_target + mean_target\n",
    "    corr_test, mae_test, mse, rmse_test, r2_test = compute_error(trues, predicted)\n",
    "    print(\"R2 Test:   %.3f\" % r2_test)\n",
    "\n",
    "    return corr_test, mae_test, rmse_test, r2_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the true labels (ground truth) and evaluate on testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "1s - loss: 0.9982\n",
      "Epoch 2/100\n",
      "0s - loss: 0.9682\n",
      "Epoch 3/100\n",
      "0s - loss: 0.9082\n",
      "Epoch 4/100\n",
      "0s - loss: 0.7853\n",
      "Epoch 5/100\n",
      "0s - loss: 0.6910\n",
      "Epoch 6/100\n",
      "0s - loss: 0.5775\n",
      "Epoch 7/100\n",
      "0s - loss: 0.4415\n",
      "Epoch 8/100\n",
      "0s - loss: 0.3370\n",
      "Epoch 9/100\n",
      "0s - loss: 0.2475\n",
      "Epoch 10/100\n",
      "0s - loss: 0.2256\n",
      "Epoch 11/100\n",
      "0s - loss: 0.1557\n",
      "Epoch 12/100\n",
      "0s - loss: 0.1254\n",
      "Epoch 13/100\n",
      "0s - loss: 0.1152\n",
      "Epoch 14/100\n",
      "0s - loss: 0.0894\n",
      "Epoch 15/100\n",
      "0s - loss: 0.0907\n",
      "Epoch 16/100\n",
      "0s - loss: 0.0866\n",
      "Epoch 17/100\n",
      "0s - loss: 0.0861\n",
      "Epoch 18/100\n",
      "0s - loss: 0.0726\n",
      "Epoch 19/100\n",
      "0s - loss: 0.0702\n",
      "Epoch 20/100\n",
      "0s - loss: 0.0594\n",
      "Epoch 21/100\n",
      "0s - loss: 0.0614\n",
      "Epoch 22/100\n",
      "0s - loss: 0.0534\n",
      "Epoch 23/100\n",
      "0s - loss: 0.0587\n",
      "Epoch 24/100\n",
      "0s - loss: 0.0748\n",
      "Epoch 25/100\n",
      "0s - loss: 0.0671\n",
      "Epoch 26/100\n",
      "0s - loss: 0.0666\n",
      "Epoch 27/100\n",
      "0s - loss: 0.0652\n",
      "Epoch 28/100\n",
      "0s - loss: 0.0512\n",
      "Epoch 29/100\n",
      "0s - loss: 0.0477\n",
      "Epoch 30/100\n",
      "0s - loss: 0.0432\n",
      "Epoch 31/100\n",
      "0s - loss: 0.0451\n",
      "Epoch 32/100\n",
      "0s - loss: 0.0391\n",
      "Epoch 33/100\n",
      "0s - loss: 0.0424\n",
      "Epoch 34/100\n",
      "0s - loss: 0.0372\n",
      "Epoch 35/100\n",
      "0s - loss: 0.0362\n",
      "Epoch 36/100\n",
      "0s - loss: 0.0388\n",
      "Epoch 37/100\n",
      "0s - loss: 0.0354\n",
      "Epoch 38/100\n",
      "0s - loss: 0.0347\n",
      "Epoch 39/100\n",
      "0s - loss: 0.0400\n",
      "Epoch 40/100\n",
      "0s - loss: 0.0387\n",
      "Epoch 41/100\n",
      "0s - loss: 0.0339\n",
      "Epoch 42/100\n",
      "0s - loss: 0.0355\n",
      "Epoch 43/100\n",
      "0s - loss: 0.0342\n",
      "Epoch 44/100\n",
      "0s - loss: 0.0362\n",
      "Epoch 45/100\n",
      "0s - loss: 0.0423\n",
      "Epoch 46/100\n",
      "0s - loss: 0.0459\n",
      "Epoch 47/100\n",
      "0s - loss: 0.0391\n",
      "Epoch 48/100\n",
      "0s - loss: 0.0352\n",
      "Epoch 49/100\n",
      "0s - loss: 0.0357\n",
      "Epoch 50/100\n",
      "0s - loss: 0.0296\n",
      "Epoch 51/100\n",
      "0s - loss: 0.0304\n",
      "Epoch 52/100\n",
      "0s - loss: 0.0278\n",
      "Epoch 53/100\n",
      "0s - loss: 0.0303\n",
      "Epoch 54/100\n",
      "0s - loss: 0.0292\n",
      "Epoch 55/100\n",
      "0s - loss: 0.0342\n",
      "Epoch 56/100\n",
      "0s - loss: 0.0347\n",
      "Epoch 57/100\n",
      "0s - loss: 0.0299\n",
      "Epoch 58/100\n",
      "0s - loss: 0.0263\n",
      "Epoch 59/100\n",
      "0s - loss: 0.0330\n",
      "Epoch 60/100\n",
      "0s - loss: 0.0317\n",
      "Epoch 61/100\n",
      "0s - loss: 0.0275\n",
      "Epoch 62/100\n",
      "0s - loss: 0.0296\n",
      "Epoch 63/100\n",
      "0s - loss: 0.0298\n",
      "Epoch 64/100\n",
      "0s - loss: 0.0276\n",
      "Epoch 65/100\n",
      "0s - loss: 0.0237\n",
      "Epoch 66/100\n",
      "0s - loss: 0.0253\n",
      "Epoch 67/100\n",
      "0s - loss: 0.0255\n",
      "Epoch 68/100\n",
      "0s - loss: 0.0285\n",
      "Epoch 69/100\n",
      "0s - loss: 0.0259\n",
      "Epoch 70/100\n",
      "0s - loss: 0.0274\n",
      "Epoch 71/100\n",
      "0s - loss: 0.0254\n",
      "Epoch 72/100\n",
      "0s - loss: 0.0259\n",
      "Epoch 73/100\n",
      "0s - loss: 0.0240\n",
      "Epoch 74/100\n",
      "0s - loss: 0.0246\n",
      "Epoch 75/100\n",
      "0s - loss: 0.0232\n",
      "Epoch 76/100\n",
      "0s - loss: 0.0267\n",
      "Epoch 77/100\n",
      "0s - loss: 0.0239\n",
      "Epoch 78/100\n",
      "0s - loss: 0.0246\n",
      "Epoch 79/100\n",
      "0s - loss: 0.0227\n",
      "Epoch 80/100\n",
      "0s - loss: 0.0234\n",
      "Epoch 81/100\n",
      "0s - loss: 0.0268\n",
      "Epoch 82/100\n",
      "0s - loss: 0.0361\n",
      "Epoch 83/100\n",
      "0s - loss: 0.0292\n",
      "Epoch 84/100\n",
      "0s - loss: 0.0259\n",
      "Epoch 85/100\n",
      "0s - loss: 0.0260\n",
      "Epoch 86/100\n",
      "0s - loss: 0.0236\n",
      "Epoch 87/100\n",
      "0s - loss: 0.0248\n",
      "Epoch 88/100\n",
      "0s - loss: 0.0236\n",
      "Epoch 89/100\n",
      "0s - loss: 0.0231\n",
      "Epoch 90/100\n",
      "0s - loss: 0.0223\n",
      "Epoch 91/100\n",
      "0s - loss: 0.0242\n",
      "Epoch 92/100\n",
      "0s - loss: 0.0252\n",
      "Epoch 93/100\n",
      "0s - loss: 0.0233\n",
      "Epoch 94/100\n",
      "0s - loss: 0.0232\n",
      "Epoch 95/100\n",
      "0s - loss: 0.0261\n",
      "Epoch 96/100\n",
      "0s - loss: 0.0260\n",
      "Epoch 97/100\n",
      "0s - loss: 0.0260\n",
      "Epoch 98/100\n",
      "0s - loss: 0.0214\n",
      "Epoch 99/100\n",
      "0s - loss: 0.0225\n",
      "Epoch 100/100\n",
      "0s - loss: 0.0242\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f5c614896d0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model, _, _ = build_base_model()\n",
    "model.fit(data_train, targets_train, batch_size=BATCH_SIZE, epochs=N_EPOCHS, shuffle=True, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2 Test:   0.453\n"
     ]
    }
   ],
   "source": [
    "corr_test, mae_test, rmse_test, r2_test = eval_model(model, data_test, targets_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the output of majority voting and evaluate on testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "0s - loss: 0.7590\n",
      "Epoch 2/100\n",
      "0s - loss: 0.7391\n",
      "Epoch 3/100\n",
      "0s - loss: 0.6960\n",
      "Epoch 4/100\n",
      "0s - loss: 0.6638\n",
      "Epoch 5/100\n",
      "0s - loss: 0.5879\n",
      "Epoch 6/100\n",
      "0s - loss: 0.4720\n",
      "Epoch 7/100\n",
      "0s - loss: 0.3754\n",
      "Epoch 8/100\n",
      "0s - loss: 0.3676\n",
      "Epoch 9/100\n",
      "0s - loss: 0.2721\n",
      "Epoch 10/100\n",
      "0s - loss: 0.2234\n",
      "Epoch 11/100\n",
      "0s - loss: 0.1791\n",
      "Epoch 12/100\n",
      "0s - loss: 0.1554\n",
      "Epoch 13/100\n",
      "0s - loss: 0.1222\n",
      "Epoch 14/100\n",
      "0s - loss: 0.1105\n",
      "Epoch 15/100\n",
      "0s - loss: 0.0839\n",
      "Epoch 16/100\n",
      "0s - loss: 0.0756\n",
      "Epoch 17/100\n",
      "0s - loss: 0.0765\n",
      "Epoch 18/100\n",
      "0s - loss: 0.0619\n",
      "Epoch 19/100\n",
      "0s - loss: 0.0580\n",
      "Epoch 20/100\n",
      "0s - loss: 0.0510\n",
      "Epoch 21/100\n",
      "0s - loss: 0.0492\n",
      "Epoch 22/100\n",
      "0s - loss: 0.0433\n",
      "Epoch 23/100\n",
      "0s - loss: 0.0437\n",
      "Epoch 24/100\n",
      "0s - loss: 0.0442\n",
      "Epoch 25/100\n",
      "0s - loss: 0.0612\n",
      "Epoch 26/100\n",
      "0s - loss: 0.0487\n",
      "Epoch 27/100\n",
      "0s - loss: 0.0471\n",
      "Epoch 28/100\n",
      "0s - loss: 0.0429\n",
      "Epoch 29/100\n",
      "0s - loss: 0.0416\n",
      "Epoch 30/100\n",
      "0s - loss: 0.0376\n",
      "Epoch 31/100\n",
      "0s - loss: 0.0334\n",
      "Epoch 32/100\n",
      "0s - loss: 0.0340\n",
      "Epoch 33/100\n",
      "0s - loss: 0.0425\n",
      "Epoch 34/100\n",
      "0s - loss: 0.0357\n",
      "Epoch 35/100\n",
      "0s - loss: 0.0365\n",
      "Epoch 36/100\n",
      "0s - loss: 0.0378\n",
      "Epoch 37/100\n",
      "0s - loss: 0.0316\n",
      "Epoch 38/100\n",
      "0s - loss: 0.0325\n",
      "Epoch 39/100\n",
      "0s - loss: 0.0304\n",
      "Epoch 40/100\n",
      "0s - loss: 0.0271\n",
      "Epoch 41/100\n",
      "0s - loss: 0.0273\n",
      "Epoch 42/100\n",
      "0s - loss: 0.0305\n",
      "Epoch 43/100\n",
      "0s - loss: 0.0317\n",
      "Epoch 44/100\n",
      "0s - loss: 0.0288\n",
      "Epoch 45/100\n",
      "0s - loss: 0.0257\n",
      "Epoch 46/100\n",
      "0s - loss: 0.0253\n",
      "Epoch 47/100\n",
      "0s - loss: 0.0259\n",
      "Epoch 48/100\n",
      "0s - loss: 0.0275\n",
      "Epoch 49/100\n",
      "0s - loss: 0.0261\n",
      "Epoch 50/100\n",
      "0s - loss: 0.0260\n",
      "Epoch 51/100\n",
      "0s - loss: 0.0245\n",
      "Epoch 52/100\n",
      "0s - loss: 0.0243\n",
      "Epoch 53/100\n",
      "0s - loss: 0.0232\n",
      "Epoch 54/100\n",
      "0s - loss: 0.0249\n",
      "Epoch 55/100\n",
      "0s - loss: 0.0256\n",
      "Epoch 56/100\n",
      "0s - loss: 0.0260\n",
      "Epoch 57/100\n",
      "0s - loss: 0.0238\n",
      "Epoch 58/100\n",
      "0s - loss: 0.0239\n",
      "Epoch 59/100\n",
      "0s - loss: 0.0228\n",
      "Epoch 60/100\n",
      "0s - loss: 0.0227\n",
      "Epoch 61/100\n",
      "0s - loss: 0.0210\n",
      "Epoch 62/100\n",
      "0s - loss: 0.0221\n",
      "Epoch 63/100\n",
      "0s - loss: 0.0240\n",
      "Epoch 64/100\n",
      "0s - loss: 0.0221\n",
      "Epoch 65/100\n",
      "0s - loss: 0.0297\n",
      "Epoch 66/100\n",
      "0s - loss: 0.0298\n",
      "Epoch 67/100\n",
      "0s - loss: 0.0234\n",
      "Epoch 68/100\n",
      "0s - loss: 0.0272\n",
      "Epoch 69/100\n",
      "0s - loss: 0.0281\n",
      "Epoch 70/100\n",
      "0s - loss: 0.0218\n",
      "Epoch 71/100\n",
      "0s - loss: 0.0224\n",
      "Epoch 72/100\n",
      "0s - loss: 0.0214\n",
      "Epoch 73/100\n",
      "0s - loss: 0.0221\n",
      "Epoch 74/100\n",
      "0s - loss: 0.0219\n",
      "Epoch 75/100\n",
      "0s - loss: 0.0206\n",
      "Epoch 76/100\n",
      "0s - loss: 0.0241\n",
      "Epoch 77/100\n",
      "0s - loss: 0.0236\n",
      "Epoch 78/100\n",
      "0s - loss: 0.0206\n",
      "Epoch 79/100\n",
      "0s - loss: 0.0218\n",
      "Epoch 80/100\n",
      "0s - loss: 0.0270\n",
      "Epoch 81/100\n",
      "0s - loss: 0.0234\n",
      "Epoch 82/100\n",
      "0s - loss: 0.0207\n",
      "Epoch 83/100\n",
      "0s - loss: 0.0235\n",
      "Epoch 84/100\n",
      "0s - loss: 0.0267\n",
      "Epoch 85/100\n",
      "0s - loss: 0.0207\n",
      "Epoch 86/100\n",
      "0s - loss: 0.0204\n",
      "Epoch 87/100\n",
      "0s - loss: 0.0206\n",
      "Epoch 88/100\n",
      "0s - loss: 0.0200\n",
      "Epoch 89/100\n",
      "0s - loss: 0.0205\n",
      "Epoch 90/100\n",
      "0s - loss: 0.0188\n",
      "Epoch 91/100\n",
      "0s - loss: 0.0203\n",
      "Epoch 92/100\n",
      "0s - loss: 0.0204\n",
      "Epoch 93/100\n",
      "0s - loss: 0.0234\n",
      "Epoch 94/100\n",
      "0s - loss: 0.0190\n",
      "Epoch 95/100\n",
      "0s - loss: 0.0179\n",
      "Epoch 96/100\n",
      "0s - loss: 0.0212\n",
      "Epoch 97/100\n",
      "0s - loss: 0.0204\n",
      "Epoch 98/100\n",
      "0s - loss: 0.0196\n",
      "Epoch 99/100\n",
      "0s - loss: 0.0181\n",
      "Epoch 100/100\n",
      "0s - loss: 0.0187\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f5c259d9ed0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model, _, _ = build_base_model()\n",
    "model.fit(data_train, targets_train_mean, batch_size=BATCH_SIZE, epochs=N_EPOCHS, shuffle=True, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2 Test:   0.301\n"
     ]
    }
   ],
   "source": [
    "corr_test, mae_test, rmse_test, r2_test = eval_model(model, data_test, targets_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model using proposed DL-B approach and evaluate on testset\n",
    "\n",
    "We start by adding a new layer (CrowdsRegression) on top of our neural network. We then require a special loss (MaskedMultiMSE) to handle the missing labels from some of the annotators."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "0s - loss: 0.0492\n",
      "Epoch 2/100\n",
      "0s - loss: 0.0474\n",
      "Epoch 3/100\n",
      "0s - loss: 0.0465\n",
      "Epoch 4/100\n",
      "0s - loss: 0.0442\n",
      "Epoch 5/100\n",
      "0s - loss: 0.0416\n",
      "Epoch 6/100\n",
      "0s - loss: 0.0375\n",
      "Epoch 7/100\n",
      "0s - loss: 0.0361\n",
      "Epoch 8/100\n",
      "0s - loss: 0.0345\n",
      "Epoch 9/100\n",
      "0s - loss: 0.0312\n",
      "Epoch 10/100\n",
      "0s - loss: 0.0289\n",
      "Epoch 11/100\n",
      "0s - loss: 0.0266\n",
      "Epoch 12/100\n",
      "0s - loss: 0.0252\n",
      "Epoch 13/100\n",
      "0s - loss: 0.0241\n",
      "Epoch 14/100\n",
      "0s - loss: 0.0236\n",
      "Epoch 15/100\n",
      "0s - loss: 0.0223\n",
      "Epoch 16/100\n",
      "0s - loss: 0.0217\n",
      "Epoch 17/100\n",
      "0s - loss: 0.0214\n",
      "Epoch 18/100\n",
      "0s - loss: 0.0209\n",
      "Epoch 19/100\n",
      "0s - loss: 0.0208\n",
      "Epoch 20/100\n",
      "0s - loss: 0.0204\n",
      "Epoch 21/100\n",
      "0s - loss: 0.0203\n",
      "Epoch 22/100\n",
      "0s - loss: 0.0200\n",
      "Epoch 23/100\n",
      "0s - loss: 0.0198\n",
      "Epoch 24/100\n",
      "0s - loss: 0.0197\n",
      "Epoch 25/100\n",
      "0s - loss: 0.0195\n",
      "Epoch 26/100\n",
      "0s - loss: 0.0194\n",
      "Epoch 27/100\n",
      "0s - loss: 0.0195\n",
      "Epoch 28/100\n",
      "0s - loss: 0.0194\n",
      "Epoch 29/100\n",
      "0s - loss: 0.0196\n",
      "Epoch 30/100\n",
      "0s - loss: 0.0193\n",
      "Epoch 31/100\n",
      "0s - loss: 0.0189\n",
      "Epoch 32/100\n",
      "0s - loss: 0.0187\n",
      "Epoch 33/100\n",
      "0s - loss: 0.0186\n",
      "Epoch 34/100\n",
      "0s - loss: 0.0185\n",
      "Epoch 35/100\n",
      "0s - loss: 0.0183\n",
      "Epoch 36/100\n",
      "0s - loss: 0.0181\n",
      "Epoch 37/100\n",
      "0s - loss: 0.0183\n",
      "Epoch 38/100\n",
      "0s - loss: 0.0181\n",
      "Epoch 39/100\n",
      "0s - loss: 0.0180\n",
      "Epoch 40/100\n",
      "0s - loss: 0.0178\n",
      "Epoch 41/100\n",
      "0s - loss: 0.0179\n",
      "Epoch 42/100\n",
      "0s - loss: 0.0177\n",
      "Epoch 43/100\n",
      "0s - loss: 0.0176\n",
      "Epoch 44/100\n",
      "0s - loss: 0.0177\n",
      "Epoch 45/100\n",
      "0s - loss: 0.0180\n",
      "Epoch 46/100\n",
      "0s - loss: 0.0175\n",
      "Epoch 47/100\n",
      "0s - loss: 0.0174\n",
      "Epoch 48/100\n",
      "0s - loss: 0.0173\n",
      "Epoch 49/100\n",
      "0s - loss: 0.0175\n",
      "Epoch 50/100\n",
      "0s - loss: 0.0174\n",
      "Epoch 51/100\n",
      "0s - loss: 0.0171\n",
      "Epoch 52/100\n",
      "0s - loss: 0.0171\n",
      "Epoch 53/100\n",
      "0s - loss: 0.0169\n",
      "Epoch 54/100\n",
      "0s - loss: 0.0170\n",
      "Epoch 55/100\n",
      "0s - loss: 0.0171\n",
      "Epoch 56/100\n",
      "0s - loss: 0.0168\n",
      "Epoch 57/100\n",
      "0s - loss: 0.0168\n",
      "Epoch 58/100\n",
      "0s - loss: 0.0169\n",
      "Epoch 59/100\n",
      "0s - loss: 0.0167\n",
      "Epoch 60/100\n",
      "0s - loss: 0.0167\n",
      "Epoch 61/100\n",
      "0s - loss: 0.0165\n",
      "Epoch 62/100\n",
      "0s - loss: 0.0165\n",
      "Epoch 63/100\n",
      "0s - loss: 0.0163\n",
      "Epoch 64/100\n",
      "0s - loss: 0.0163\n",
      "Epoch 65/100\n",
      "0s - loss: 0.0163\n",
      "Epoch 66/100\n",
      "0s - loss: 0.0163\n",
      "Epoch 67/100\n",
      "0s - loss: 0.0163\n",
      "Epoch 68/100\n",
      "0s - loss: 0.0164\n",
      "Epoch 69/100\n",
      "0s - loss: 0.0162\n",
      "Epoch 70/100\n",
      "0s - loss: 0.0162\n",
      "Epoch 71/100\n",
      "0s - loss: 0.0161\n",
      "Epoch 72/100\n",
      "0s - loss: 0.0162\n",
      "Epoch 73/100\n",
      "0s - loss: 0.0161\n",
      "Epoch 74/100\n",
      "0s - loss: 0.0161\n",
      "Epoch 75/100\n",
      "0s - loss: 0.0159\n",
      "Epoch 76/100\n",
      "0s - loss: 0.0159\n",
      "Epoch 77/100\n",
      "0s - loss: 0.0160\n",
      "Epoch 78/100\n",
      "0s - loss: 0.0159\n",
      "Epoch 79/100\n",
      "0s - loss: 0.0158\n",
      "Epoch 80/100\n",
      "0s - loss: 0.0158\n",
      "Epoch 81/100\n",
      "0s - loss: 0.0158\n",
      "Epoch 82/100\n",
      "0s - loss: 0.0157\n",
      "Epoch 83/100\n",
      "0s - loss: 0.0156\n",
      "Epoch 84/100\n",
      "0s - loss: 0.0157\n",
      "Epoch 85/100\n",
      "0s - loss: 0.0156\n",
      "Epoch 86/100\n",
      "0s - loss: 0.0156\n",
      "Epoch 87/100\n",
      "0s - loss: 0.0156\n",
      "Epoch 88/100\n",
      "0s - loss: 0.0157\n",
      "Epoch 89/100\n",
      "0s - loss: 0.0155\n",
      "Epoch 90/100\n",
      "0s - loss: 0.0155\n",
      "Epoch 91/100\n",
      "0s - loss: 0.0155\n",
      "Epoch 92/100\n",
      "0s - loss: 0.0154\n",
      "Epoch 93/100\n",
      "0s - loss: 0.0154\n",
      "Epoch 94/100\n",
      "0s - loss: 0.0154\n",
      "Epoch 95/100\n",
      "0s - loss: 0.0153\n",
      "Epoch 96/100\n",
      "0s - loss: 0.0153\n",
      "Epoch 97/100\n",
      "0s - loss: 0.0153\n",
      "Epoch 98/100\n",
      "0s - loss: 0.0154\n",
      "Epoch 99/100\n",
      "0s - loss: 0.0152\n",
      "Epoch 100/100\n",
      "0s - loss: 0.0152\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f5c255bfad0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model, sequence_input, preds = build_base_model()\n",
    "\n",
    "# add crowds layer on top of the base model\n",
    "ma_preds = CrowdsRegression(N_ANNOT, conn_type=\"B\")(preds)\n",
    "\n",
    "# instantiate specialized masked loss to handle missing answers\n",
    "loss = MaskedMultiMSE().loss\n",
    "\n",
    "# compile model with masked loss and train\n",
    "model = Model(sequence_input, ma_preds)\n",
    "model.compile(optimizer='adam', loss=loss)\n",
    "\n",
    "# train model\n",
    "model.fit(data_train, answers, batch_size=BATCH_SIZE, epochs=N_EPOCHS, shuffle=True, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before evaluating our model, we need to remove the crowds layer used during training in order to expose the aggregation (bottleneck) layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2 Test:   0.400\n"
     ]
    }
   ],
   "source": [
    "# save weights from crowds layer for later\n",
    "weights = model.layers[5].get_weights()\n",
    "\n",
    "# skip CrowdsLayer for predictions\n",
    "model = Model(sequence_input, preds) \n",
    "model.compile(loss=\"mse\", optimizer='adam')\n",
    "\n",
    "# evaluate model\n",
    "corr_test, mae_test, rmse_test, r2_test = eval_model(model, data_test, targets_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}