demo-conll-ner-mturk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.cm as cm\n",
    "from matplotlib import pyplot as plt \n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense, Conv1D\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.wrappers import TimeDistributed\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers.embeddings import Embedding\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "from conlleval import conlleval\n",
    "\n",
    "# packages for learning from crowds\n",
    "from crowd_layer.crowd_layers import CrowdsClassification, MaskedMultiSequenceCrossEntropy\n",
    "from crowd_layer.crowd_aggregators import CrowdsCategoricalAggregator\n",
    "\n",
    "# prevent tensorflow from allocating the entire GPU memory at once\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_RUNS = 30\n",
    "DATA_PATH = \"/home/fmpr/datasets/deep-crowds-datasets/ner-mturk/\"\n",
    "EMBEDDING_DIM = 300\n",
    "BATCH_SIZE = 64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load indexing word vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "f = open(\"/home/fmpr/datasets/glove.6B/glove.6B.%dd.txt\" % (EMBEDDING_DIM,))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Found %s word vectors' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_conll(filename):\n",
    "    raw = open(filename, 'r').readlines()\n",
    "    all_x = []\n",
    "    point = []\n",
    "    for line in raw:\n",
    "        stripped_line = line.strip().split(' ')\n",
    "        point.append(stripped_line)\n",
    "        if line == '\\n':\n",
    "            if len(point[:-1]) > 0:\n",
    "                all_x.append(point[:-1])\n",
    "            point = []\n",
    "    all_x = all_x\n",
    "    return all_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers data size: 5985\n",
      "Majority voting data size: 5985\n",
      "Ground truth data size: 5985\n",
      "Test data size: 3250\n",
      "Total sequences: 9235\n"
     ]
    }
   ],
   "source": [
    "all_answers = read_conll(DATA_PATH+'answers.txt')\n",
    "all_mv = read_conll(DATA_PATH+'mv.txt')\n",
    "all_ground_truth = read_conll(DATA_PATH+'ground_truth.txt')\n",
    "all_test = read_conll(DATA_PATH+'testset.txt')\n",
    "all_docs = all_ground_truth + all_test\n",
    "print \"Answers data size:\", len(all_answers)\n",
    "print \"Majority voting data size:\", len(all_mv)\n",
    "print \"Ground truth data size:\", len(all_ground_truth)\n",
    "print \"Test data size:\", len(all_test)\n",
    "print \"Total sequences:\", len(all_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num annnotators: 47\n",
      "Labels: ['I-LOC', 'B-ORG', 'I-PER', 'O', 'I-MISC', 'B-MISC', 'I-ORG', 'B-LOC', 'B-PER']\n",
      "Input sequence length range:  109 1\n",
      "Max label: 10\n",
      "Maximum sequence length: 109\n"
     ]
    }
   ],
   "source": [
    "X_train = [[c[0] for c in x] for x in all_answers]\n",
    "y_answers = [[c[1:] for c in y] for y in all_answers]\n",
    "y_mv = [[c[1] for c in y] for y in all_mv]\n",
    "y_ground_truth = [[c[1] for c in y] for y in all_ground_truth]\n",
    "X_test = [[c[0] for c in x] for x in all_test]\n",
    "y_test = [[c[1] for c in y] for y in all_test]\n",
    "X_all = [[c[0] for c in x] for x in all_docs]\n",
    "y_all = [[c[1] for c in y] for y in all_docs]\n",
    "\n",
    "N_ANNOT = len(y_answers[0][0])\n",
    "print \"Num annnotators:\", N_ANNOT\n",
    "\n",
    "lengths = [len(x) for x in all_docs]\n",
    "all_text = [c for x in X_all for c in x]\n",
    "words = list(set(all_text))\n",
    "word2ind = {word: index for index, word in enumerate(words)}\n",
    "ind2word = {index: word for index, word in enumerate(words)}\n",
    "labels = list(set([c for x in y_all for c in x]))\n",
    "print \"Labels:\", labels\n",
    "label2ind = {label: (index + 1) for index, label in enumerate(labels)}\n",
    "ind2label = {(index + 1): label for index, label in enumerate(labels)}\n",
    "ind2label[0] = \"O\" # padding index\n",
    "print 'Input sequence length range: ', max(lengths), min(lengths)\n",
    "\n",
    "max_label = max(label2ind.values()) + 1\n",
    "print \"Max label:\", max_label\n",
    "\n",
    "maxlen = max([len(x) for x in X_all])\n",
    "print 'Maximum sequence length:', maxlen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_words = len(word2ind)\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "for word, i in word2ind.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert data to one-hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def encode(x, n):\n",
    "    result = np.zeros(n)\n",
    "    result[x] = 1\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_enc = [[word2ind[c] for c in x] for x in X_train]\n",
    "y_ground_truth_enc = [[0] * (maxlen - len(ey)) + [label2ind[c] for c in ey] for ey in y_ground_truth]\n",
    "y_ground_truth_enc = [[encode(c, max_label) for c in ey] for ey in y_ground_truth_enc]\n",
    "y_mv_enc = [[0] * (maxlen - len(ey)) + [label2ind[c] for c in ey] for ey in y_mv]\n",
    "y_mv_enc = [[encode(c, max_label) for c in ey] for ey in y_mv_enc]\n",
    "\n",
    "y_answers_enc = []\n",
    "for r in xrange(N_ANNOT):\n",
    "    annot_answers = []\n",
    "    for i in xrange(len(y_answers)):\n",
    "        seq = []\n",
    "        for j in xrange(len(y_answers[i])):\n",
    "            #enc = -1*np.ones(max_label)\n",
    "            enc = -1\n",
    "            if y_answers[i][j][r] != \"?\":\n",
    "                enc = label2ind[y_answers[i][j][r]]\n",
    "            seq.append(enc)\n",
    "        annot_answers.append(seq)\n",
    "    y_answers_enc.append(annot_answers)\n",
    "\n",
    "X_test_enc = [[word2ind[c] for c in x] for x in X_test]\n",
    "y_test_enc = [[0] * (maxlen - len(ey)) + [label2ind[c] for c in ey] for ey in y_test]\n",
    "y_test_enc = [[encode(c, max_label) for c in ey] for ey in y_test_enc]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pad sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training and testing tensor shapes:\n",
      "(5985, 109) (3250, 109) (5985, 109, 10) (3250, 109, 10)\n",
      "Answers shape: (5985, 109, 47)\n",
      "Num classes: 10\n"
     ]
    }
   ],
   "source": [
    "# pad sequences\n",
    "X_train_enc = pad_sequences(X_train_enc, maxlen=maxlen)\n",
    "y_ground_truth_enc = pad_sequences(y_ground_truth_enc, maxlen=maxlen)\n",
    "X_test_enc = pad_sequences(X_test_enc, maxlen=maxlen)\n",
    "y_test_enc = pad_sequences(y_test_enc, maxlen=maxlen)\n",
    "\n",
    "y_answers_enc_padded = []\n",
    "for r in xrange(N_ANNOT):\n",
    "    padded_answers = pad_sequences(y_answers_enc[r], maxlen=maxlen)\n",
    "    y_answers_enc_padded.append(padded_answers)\n",
    "\n",
    "y_answers_enc_padded = np.array(y_answers_enc_padded)\n",
    "y_answers_enc = np.transpose(np.array(y_answers_enc_padded), [1, 2, 0])\n",
    "\n",
    "n_train = len(X_train_enc)\n",
    "n_test = len(X_test_enc)\n",
    "\n",
    "print 'Training and testing tensor shapes:'\n",
    "print X_train_enc.shape, X_test_enc.shape, y_ground_truth_enc.shape, y_test_enc.shape\n",
    "\n",
    "print \"Answers shape:\", y_answers_enc.shape\n",
    "\n",
    "N_CLASSES = len(label2ind) + 1\n",
    "print \"Num classes:\", N_CLASSES"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the base deep learning model\n",
    "\n",
    "Here we shall use features representation produced by the VGG16 network as the input. Our base model is then simply composed by one densely-connected layer with 128 hidden units and an output dense layer. We use 50% dropout between the two dense layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_base_model():\n",
    "    base_model = Sequential()\n",
    "    base_model.add(Embedding(num_words,\n",
    "                        300,\n",
    "                        weights=[embedding_matrix],\n",
    "                        input_length=maxlen,\n",
    "                        trainable=True))\n",
    "    base_model.add(Conv1D(512, 5, padding=\"same\", activation=\"relu\"))\n",
    "    base_model.add(Dropout(0.5))\n",
    "    base_model.add(GRU(50, return_sequences=True))\n",
    "    base_model.add(TimeDistributed(Dense(N_CLASSES, activation='softmax')))\n",
    "    base_model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "    return base_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary functions for evaluating the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score(yh, pr):\n",
    "    coords = [np.where(yhh > 0)[0][0] for yhh in yh]\n",
    "    yh = [yhh[co:] for yhh, co in zip(yh, coords)]\n",
    "    ypr = [prr[co:] for prr, co in zip(pr, coords)]\n",
    "    fyh = [c for row in yh for c in row]\n",
    "    fpr = [c for row in ypr for c in row]\n",
    "    return fyh, fpr\n",
    "\n",
    "def eval_model(model):\n",
    "    pr_test = model.predict(X_test_enc, verbose=2)\n",
    "    pr_test = np.argmax(pr_test, axis=2)\n",
    "\n",
    "    yh = y_test_enc.argmax(2)\n",
    "    fyh, fpr = score(yh, pr_test)\n",
    "    print 'Testing accuracy:', accuracy_score(fyh, fpr)\n",
    "    print 'Testing confusion matrix:'\n",
    "    print confusion_matrix(fyh, fpr)\n",
    "\n",
    "    preds_test = []\n",
    "    for i in xrange(len(pr_test)):\n",
    "        row = pr_test[i][-len(y_test[i]):]\n",
    "        row[np.where(row == 0)] = 1\n",
    "        preds_test.append(row)\n",
    "    preds_test = [ list(map(lambda x: ind2label[x], y)) for y in preds_test]\n",
    "\n",
    "    results_test = conlleval(preds_test, y_test, X_test, 'r_test.txt')\n",
    "    print \"Results for testset:\", results_test\n",
    "\n",
    "    return results_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the true labels (ground truth) and evaluate on testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "14s - loss: 0.1156\n",
      "Epoch 2/20\n",
      "13s - loss: 0.0358\n",
      "Epoch 3/20\n",
      "13s - loss: 0.0200\n",
      "Epoch 4/20\n",
      "13s - loss: 0.0107\n",
      "Epoch 5/20\n",
      "13s - loss: 0.0059\n",
      "Epoch 6/20\n",
      "13s - loss: 0.0034\n",
      "Epoch 7/20\n",
      "13s - loss: 0.0023\n",
      "Epoch 8/20\n",
      "13s - loss: 0.0015\n",
      "Epoch 9/20\n",
      "13s - loss: 0.0012\n",
      "Epoch 10/20\n",
      "13s - loss: 8.8251e-04\n",
      "Epoch 11/20\n",
      "13s - loss: 7.1133e-04\n",
      "Epoch 12/20\n",
      "13s - loss: 5.9820e-04\n",
      "Epoch 13/20\n",
      "13s - loss: 4.8791e-04\n",
      "Epoch 14/20\n",
      "13s - loss: 4.0111e-04\n",
      "Epoch 15/20\n",
      "12s - loss: 3.5680e-04\n",
      "Epoch 16/20\n",
      "13s - loss: 2.7868e-04\n",
      "Epoch 17/20\n",
      "13s - loss: 2.3333e-04\n",
      "Epoch 18/20\n",
      "13s - loss: 2.0176e-04\n",
      "Epoch 19/20\n",
      "13s - loss: 1.8314e-04\n",
      "Epoch 20/20\n",
      "13s - loss: 1.8913e-04\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f4cb47b5e50>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "model.fit(X_train_enc, y_ground_truth_enc, batch_size=BATCH_SIZE, epochs=20, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 0.947860285814\n",
      "Testing confusion matrix:\n",
      "[[    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   150     3    15    20    11     0    53     4     1]\n",
      " [    0     0  1100     7   126     2    44    12    23    27]\n",
      " [    0     2     5  1138   111     0     2    37     0    12]\n",
      " [    1   123   217    97 42010    33    97    94    33    58]\n",
      " [    0     6    12     5    58   216    19    19     4     6]\n",
      " [    0     1    74     0   118     7   680     2    26    11]\n",
      " [    0     8    32    15    82    20     3   580    10     1]\n",
      " [    0     2   204     4   114     0    67     7  1424    15]\n",
      " [    0     1   211    26   159     2    30     9    18  1386]]\n",
      "Results for testset: {'p': 70.28, 'r': 72.45, 'f1': 71.35}\n"
     ]
    }
   ],
   "source": [
    "results_test = eval_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the output of majority voting and evaluate on testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "13s - loss: 0.1046\n",
      "Epoch 2/10\n",
      "13s - loss: 0.0382\n",
      "Epoch 3/10\n",
      "13s - loss: 0.0267\n",
      "Epoch 4/10\n",
      "13s - loss: 0.0204\n",
      "Epoch 5/10\n",
      "13s - loss: 0.0161\n",
      "Epoch 6/10\n",
      "13s - loss: 0.0127\n",
      "Epoch 7/10\n",
      "13s - loss: 0.0101\n",
      "Epoch 8/10\n",
      "13s - loss: 0.0082\n",
      "Epoch 9/10\n",
      "13s - loss: 0.0070\n",
      "Epoch 10/10\n",
      "13s - loss: 0.0060\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f4c884298d0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "model.fit(X_train_enc, y_mv_enc, batch_size=BATCH_SIZE, epochs=10, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 0.910108640629\n",
      "Testing confusion matrix:\n",
      "[[    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   122     0     6    67     1     1    40    16     4]\n",
      " [    0     0   506     6   594     1    14    16   177    27]\n",
      " [    0     0     2   942   332     3     0    14     1    13]\n",
      " [    2    39    49    33 42381    40    89    48    46    36]\n",
      " [    0     6     4     4   200    93     9    21     2     6]\n",
      " [    0     1    57     0   534     1   266     4    46    10]\n",
      " [    0    44     7    27   371     6     1   264    26     5]\n",
      " [    0     7    98     1   581     0    14     4  1118    14]\n",
      " [    0     1    46    12   693     1     2     7    27  1053]]\n",
      "Results for testset: {'p': 64.48, 'r': 44.54, 'f1': 52.68}\n"
     ]
    }
   ],
   "source": [
    "results_test = eval_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model using proposed DL-MW approach and evaluate on testset\n",
    "\n",
    "We start by pre-training the base model for a few iteration using the output of majority voting, as this improves the stability of the crowds layers. We then add a new layer (CrowdsClassification) on top of our neural network. Finally, we make use of a special loss (MaskedMultiSequenceCrossEntropy) to handle the missing labels from some of the annotators (encoded as \"-1\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "13s - loss: 0.1023\n",
      "Epoch 2/5\n",
      "13s - loss: 0.0364\n",
      "Epoch 3/5\n",
      "13s - loss: 0.0258\n",
      "Epoch 4/5\n",
      "13s - loss: 0.0194\n",
      "Epoch 5/5\n",
      "13s - loss: 0.0148\n",
      "Epoch 1/30\n",
      "14s - loss: 0.0177\n",
      "Epoch 2/30\n",
      "13s - loss: 0.0163\n",
      "Epoch 3/30\n",
      "13s - loss: 0.0150\n",
      "Epoch 4/30\n",
      "13s - loss: 0.0138\n",
      "Epoch 5/30\n",
      "13s - loss: 0.0128\n",
      "Epoch 6/30\n",
      "13s - loss: 0.0118\n",
      "Epoch 7/30\n",
      "13s - loss: 0.0109\n",
      "Epoch 8/30\n",
      "13s - loss: 0.0101\n",
      "Epoch 9/30\n",
      "13s - loss: 0.0093\n",
      "Epoch 10/30\n",
      "13s - loss: 0.0087\n",
      "Epoch 11/30\n",
      "13s - loss: 0.0081\n",
      "Epoch 12/30\n",
      "13s - loss: 0.0076\n",
      "Epoch 13/30\n",
      "13s - loss: 0.0071\n",
      "Epoch 14/30\n",
      "13s - loss: 0.0066\n",
      "Epoch 15/30\n",
      "13s - loss: 0.0063\n",
      "Epoch 16/30\n",
      "13s - loss: 0.0059\n",
      "Epoch 17/30\n",
      "13s - loss: 0.0056\n",
      "Epoch 18/30\n",
      "13s - loss: 0.0053\n",
      "Epoch 19/30\n",
      "13s - loss: 0.0050\n",
      "Epoch 20/30\n",
      "13s - loss: 0.0048\n",
      "Epoch 21/30\n",
      "13s - loss: 0.0046\n",
      "Epoch 22/30\n",
      "13s - loss: 0.0044\n",
      "Epoch 23/30\n",
      "13s - loss: 0.0042\n",
      "Epoch 24/30\n",
      "13s - loss: 0.0040\n",
      "Epoch 25/30\n",
      "13s - loss: 0.0039\n",
      "Epoch 26/30\n",
      "13s - loss: 0.0037\n",
      "Epoch 27/30\n",
      "13s - loss: 0.0036\n",
      "Epoch 28/30\n",
      "13s - loss: 0.0035\n",
      "Epoch 29/30\n",
      "13s - loss: 0.0034\n",
      "Epoch 30/30\n",
      "13s - loss: 0.0033\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f4d325cdf10>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "\n",
    "# pre-train base model for a few iterations using the output of majority voting\n",
    "model.fit(X_train_enc, y_mv_enc, batch_size=BATCH_SIZE, epochs=5, verbose=2)\n",
    "\n",
    "# add crowds layer on top of the base model\n",
    "model.add(CrowdsClassification(N_CLASSES, N_ANNOT, conn_type=\"MW\"))\n",
    "\n",
    "# instantiate specialized masked loss to handle missing answers\n",
    "loss = MaskedMultiSequenceCrossEntropy(N_CLASSES).loss\n",
    "\n",
    "# compile model with masked loss and train\n",
    "model.compile(optimizer='adam', loss=loss)\n",
    "model.fit(X_train_enc, y_answers_enc, batch_size=BATCH_SIZE, epochs=30, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before evaluating our model, we need to remove the crowds layer used during training in order to expose the aggregation (bottleneck) layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing accuracy: 0.926891476189\n",
      "Testing confusion matrix:\n",
      "[[  143     2     8    41     8     0    37    17     1]\n",
      " [    1   689     5   333     2    32    19   226    34]\n",
      " [    1     4  1156   109     2     1    17     1    16]\n",
      " [  138    84    56 41883   123   215   132    85    47]\n",
      " [    7     6     8    96   174    12    27     5    10]\n",
      " [    1    63     0   327     9   433     4    58    24]\n",
      " [   56    17    29   227    14     4   371    26     7]\n",
      " [    1    80     2   269     0    37     6  1426    16]\n",
      " [    1    40    16   402     3     8     7    33  1332]]\n",
      "Results for testset: {'p': 65.21, 'r': 59.27, 'f1': 62.1}\n"
     ]
    }
   ],
   "source": [
    "# save weights from crowds layer for later\n",
    "weights = model.layers[5].get_weights()\n",
    "\n",
    "# remove crowds layer before making predictions\n",
    "model.pop() \n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "results_test = eval_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}