NeuralNetwork.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports and Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/iker/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available GPUs:['/job:localhost/replica:0/task:0/device:GPU:0']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "from sklearn.datasets.base import Bunch\n",
    "import sklearn.metrics\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "import regex \n",
    "import string \n",
    "from tqdm import tqdm_notebook\n",
    "from embedding import load_embedding\n",
    "from utils import normalize_questions, perf_measure, printTrace\n",
    "\n",
    "\n",
    "\n",
    "from keras.layers import LSTM, GRU, Dense, Dropout, Flatten, MaxPooling1D, Convolution1D, Input, concatenate, Lambda, Bidirectional, Input, Dense, Embedding, Conv2D, MaxPool2D, Reshape, Flatten, Dropout, Concatenate\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.models import Sequential, load_model, model_from_config\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "print('Available GPUs:' + str(K.tensorflow_backend._get_available_gpus()))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Embedding_path = 'Embeddings/esTech_enTech.vec'\n",
    "Dataset_path = 'Dataset/'\n",
    "Save_path = 'Weights/'\n",
    "BATCH_SIZE = 1024\n",
    "EMB_DIMS = 300\n",
    "MAX_WORDS = 25\n",
    "EPOCHS = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<2019-06-18 02:11:39.275751>  Loading embedding Embeddings/esTech_enTech.enesJ.vec: 99%\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b23f6dab44cc418ea9579cf86b2c5b98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error in sentence: []\n",
      "Error in line: ?\tGiven that PACER makes money from usage, why does the legal profession tolerate its absurd awfulness?\t0\n",
      "Error in sentence: []\n",
      "Error in line: What is this?\to\t0\n",
      "Error in line: Mathematical Puzzles: What is () + () + () = 30 using 1,3,5,7,9,11,13,15?\tHow do I Simplify the following matrices:\n",
      "Error in line: 1. -5 [-7 0 0 5]; 2. 3 [6 4 5 -5 3 1]?\t0\n",
      "Error in sentence: []\n",
      "Error in line: FROM WHICH website CAN WE DOWNLOAD PORNOGRAPHY SAFELY AND free?\tمين عايز هامبورجر و مين عايز سوسيس؟?\t0\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f1e4b281d9f402aa3c32108cf31bade",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error in sentence: []\n",
      "Error in line: …..\tWhat do I do with this guy?\t0\n",
      "Error in sentence: []\n",
      "Error in line: no\tI have a BS and MPH and hate my job. I found that I love working with website design but is it worth going back to school?\t0\n",
      "Error in sentence: []\n",
      "Error in line: What is the psychological explanation to the existence of Rudalis in time of Rajas in India?\tماذا علي ان افعل لكي اشعر بالسعادة و الرضى ؟\t0\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13f20d0bb73f44b588f7e02eb3f4b5df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def pad_sentence(sentence,dims):\n",
    "    try:\n",
    "        if len(sentence) >= MAX_WORDS:\n",
    "            return sentence[:MAX_WORDS]\n",
    "        else:\n",
    "            while(len(sentence) < MAX_WORDS):\n",
    "                sentence = np.concatenate((sentence,[np.zeros(dims)]),axis=0)            \n",
    "            return sentence\n",
    "    except:\n",
    "        print('Error in sentence: ' + str(sentence))\n",
    "        raise ValueError('ReadingError')\n",
    "\n",
    "def sentence_to_vectors(sentence,emb,default_vec,prefix=None):\n",
    "    words = normalize_questions(sentence)\n",
    "    if len(sentence)>0:\n",
    "        vectors = []\n",
    "        for word in words:\n",
    "            try:\n",
    "                if prefix:\n",
    "                    vectors.append(emb.word_to_vector(prefix+'/'+word))\n",
    "                else:\n",
    "                    vectors.append(emb.word_to_vector(word))\n",
    "            except KeyError as r:\n",
    "                vectors.append(default_vec)\n",
    "                #print('Missing word: ' + str(word))\n",
    "        return pad_sentence(vectors, emb.dims)\n",
    "    \n",
    "    else:\n",
    "        print('Error in sentence: ' + str(sentence))\n",
    "        raise ValueError('ReadingError')\n",
    "\n",
    "def dataset_to_vectors(dataset_path, emb, default_vec, prefix = None):\n",
    "    datasetA = []\n",
    "    datasetB = []\n",
    "    golds = []\n",
    "\n",
    "    with open(dataset_path,'r') as file:\n",
    "        for line in tqdm_notebook(file):\n",
    "            line = line.rstrip()\n",
    "            try:\n",
    "                q1,q2,g = line.split('\\t')\n",
    "                #print(q1)\n",
    "                #print(q2)\n",
    "                #print(g)\n",
    "                qA = sentence_to_vectors(q1, emb, default_vec, prefix)\n",
    "                qB = sentence_to_vectors(q2, emb, default_vec, prefix)\n",
    "                datasetA.append(qA)\n",
    "                datasetB.append(qB)\n",
    "                golds.append(int(g))\n",
    "            except:\n",
    "                print('Error in line: ' + line)\n",
    "                continue\n",
    "            \n",
    "    return Bunch(X=[np.asarray(datasetA),np.asarray(datasetB)], y=golds)\n",
    "\n",
    "\n",
    "emb = load_embedding(Embedding_path, normalize_dimensionwise=False, length_normalize=False, lower=False)\n",
    "EMB_DIMS = emb.dims\n",
    "default_vec = np.mean(emb.vectors, axis=0)\n",
    "\n",
    "train_set = dataset_to_vectors(Dataset_path+'train_set.csv', emb, default_vec, 'en')\n",
    "dev_set = dataset_to_vectors(Dataset_path+'dev_set.csv', emb, default_vec, 'en')\n",
    "test_set = dataset_to_vectors(Dataset_path+'test_set_corr.csv', emb, default_vec, 'es')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EVALUATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, dataset):\n",
    "    \n",
    "    gold_scores = dataset.y\n",
    "    scores = [0 if x <0.5 else 1 for x in model.predict(dataset.X, batch_size=BATCH_SIZE)]\n",
    "    \n",
    "    result = sklearn.metrics.log_loss(gold_scores, scores)\n",
    "    TP, FP, TN, FN = perf_measure(gold_scores, scores)\n",
    "    acc = np.sum(np.array(gold_scores) == np.array(scores)) / len(gold_scores)\n",
    "    #print('======================================')\n",
    "    print('Log Loss: ' + str(result))\n",
    "    print('Acc: ' + str(acc))\n",
    "    print('TP: ' + str(TP) + '\\tFP: ' + str(FP) + '\\tTN: ' + str(TN) + '\\tFN: ' + str(FN))\n",
    "    #print('======================================')\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model_concat_LSTM():\n",
    "    \n",
    "    inputA = Input(shape=(MAX_WORDS,EMB_DIMS))\n",
    "    inputB = Input(shape=(MAX_WORDS,EMB_DIMS))\n",
    "    A = Model(inputs=inputA, outputs=inputA)\n",
    "    B = Model(inputs=inputB, outputs=inputB)\n",
    "    \n",
    "    AB = concatenate([A.output, B.output])\n",
    "    \n",
    "    NN = Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2))(AB)\n",
    "    NN = Dense(1, activation='sigmoid')(NN)\n",
    "    \n",
    "    model = Model(inputs=[A.input, B.input], outputs=NN)\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    \n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model_LSTM_concat():\n",
    "    \n",
    "    inputA = Input(shape=(MAX_WORDS,EMB_DIMS))\n",
    "    inputB = Input(shape=(MAX_WORDS,EMB_DIMS))\n",
    "    \n",
    "    A = Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2))(inputA)\n",
    "    B = Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2))(inputB)\n",
    "    \n",
    "    A = Model(inputs=inputA, outputs=A)\n",
    "    B = Model(inputs=inputB, outputs=B)\n",
    "    \n",
    "    AB = concatenate([A.output, B.output])\n",
    "    \n",
    "    NN = Dense(256, activation='sigmoid')(AB)\n",
    "    NN = Dense(1, activation='sigmoid')(NN)\n",
    "    \n",
    "    model = Model(inputs=[A.input, B.input], outputs=NN)\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    \n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model_perceptron():\n",
    "    \n",
    "    inputA = Input(shape=(MAX_WORDS,EMB_DIMS))\n",
    "    inputB = Input(shape=(MAX_WORDS,EMB_DIMS))\n",
    "    A = Model(inputs=inputA, outputs=inputA)\n",
    "    B = Model(inputs=inputB, outputs=inputB)\n",
    "    \n",
    "    AB = concatenate([A.output, B.output])\n",
    "    \n",
    "    NN = Dense(300, activation='sigmoid')(AB)\n",
    "    NN = Dense(1, activation='sigmoid')(NN)\n",
    "    \n",
    "    model = Model(inputs=[A.input, B.input], outputs=NN)\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RUN BASIC PERCEPTRON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 25, 300)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 25, 300)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 25, 600)      0           input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 256)          746496      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            257         bidirectional_1[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 746,753\n",
      "Trainable params: 746,753\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "<2019-06-18 02:16:41.851092>  Epoch 1 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 9s 142us/step - loss: 0.6300 - acc: 0.6372\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 11.056615409745104\n",
      "Acc: 0.6798812542519637\n",
      "TP: 2936\tFP: 2136\tTN: 8057\tFN: 3040\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 18.996366997072386\n",
      "Acc: 0.45\n",
      "TP: 0\tFP: 5\tTN: 45\tFN: 50\n",
      "<2019-06-18 02:16:54.778060>  Epoch 2 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 95us/step - loss: 0.5936 - acc: 0.6789\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.684879652086135\n",
      "Acc: 0.6906425876677593\n",
      "TP: 2049\tFP: 1075\tTN: 9118\tFN: 3927\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 18.305575477225567\n",
      "Acc: 0.47\n",
      "TP: 0\tFP: 3\tTN: 47\tFN: 50\n",
      "<2019-06-18 02:17:01.950823>  Epoch 3 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 95us/step - loss: 0.5790 - acc: 0.6913\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.353816245804587\n",
      "Acc: 0.700228832951945\n",
      "TP: 2887\tFP: 1758\tTN: 8435\tFN: 3089\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:17:09.068028>  Epoch 4 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 95us/step - loss: 0.5707 - acc: 0.6967\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.571650891633615\n",
      "Acc: 0.6939204650875131\n",
      "TP: 1801\tFP: 774\tTN: 9419\tFN: 4175\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 18.99638298902099\n",
      "Acc: 0.45\n",
      "TP: 2\tFP: 7\tTN: 43\tFN: 48\n",
      "<2019-06-18 02:17:16.218902>  Epoch 5 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 94us/step - loss: 0.5608 - acc: 0.7036\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.10173498843858\n",
      "Acc: 0.7075267487166801\n",
      "TP: 2597\tFP: 1350\tTN: 8843\tFN: 3379\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.96021969717367\n",
      "Acc: 0.48\n",
      "TP: 5\tFP: 7\tTN: 43\tFN: 45\n",
      "<2019-06-18 02:17:23.338969>  Epoch 6 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 94us/step - loss: 0.5549 - acc: 0.7071\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.950077942264215\n",
      "Acc: 0.7119178675242749\n",
      "TP: 2806\tFP: 1488\tTN: 8705\tFN: 3170\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.26946016122406\n",
      "Acc: 0.5\n",
      "TP: 9\tFP: 9\tTN: 41\tFN: 41\n",
      "<2019-06-18 02:17:30.469166>  Epoch 7 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 95us/step - loss: 0.5483 - acc: 0.7134\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.824047105175694\n",
      "Acc: 0.7155668254066423\n",
      "TP: 2859\tFP: 1482\tTN: 8711\tFN: 3117\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.960251681070876\n",
      "Acc: 0.48\n",
      "TP: 9\tFP: 11\tTN: 39\tFN: 41\n",
      "<2019-06-18 02:17:37.670070>  Epoch 8 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 94us/step - loss: 0.5399 - acc: 0.7200\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.78771269787487\n",
      "Acc: 0.7166182200507143\n",
      "TP: 2461\tFP: 1067\tTN: 9126\tFN: 3515\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92406440130065\n",
      "Acc: 0.51\n",
      "TP: 9\tFP: 8\tTN: 42\tFN: 41\n",
      "<2019-06-18 02:17:44.817640>  Epoch 9 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.5354 - acc: 0.7237\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.589104384947367\n",
      "Acc: 0.7223699672212258\n",
      "TP: 3565\tFP: 2078\tTN: 8115\tFN: 2411\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887869125556122\n",
      "Acc: 0.54\n",
      "TP: 8\tFP: 4\tTN: 46\tFN: 42\n",
      "<2019-06-18 02:17:51.701453>  Epoch 10 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.5265 - acc: 0.7291\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.418186339670811\n",
      "Acc: 0.7273177067227411\n",
      "TP: 3055\tFP: 1488\tTN: 8705\tFN: 2921\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887869125556122\n",
      "Acc: 0.54\n",
      "TP: 8\tFP: 4\tTN: 46\tFN: 42\n",
      "<2019-06-18 02:17:58.645273>  Epoch 11 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 95us/step - loss: 0.5174 - acc: 0.7358\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.330629136244237\n",
      "Acc: 0.7298534232172676\n",
      "TP: 3568\tFP: 1960\tTN: 8233\tFN: 2408\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887837141658915\n",
      "Acc: 0.54\n",
      "TP: 4\tFP: 0\tTN: 50\tFN: 46\n",
      "<2019-06-18 02:18:05.787108>  Epoch 12 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 94us/step - loss: 0.5101 - acc: 0.7418\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.210976366788682\n",
      "Acc: 0.7333168408683283\n",
      "TP: 3006\tFP: 1342\tTN: 8851\tFN: 2970\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:18:12.873081>  Epoch 13 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 93us/step - loss: 0.5017 - acc: 0.7467\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.074267299698592\n",
      "Acc: 0.7372750324695405\n",
      "TP: 3111\tFP: 1383\tTN: 8810\tFN: 2865\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:18:19.923820>  Epoch 14 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 95us/step - loss: 0.4968 - acc: 0.7503\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.005926439769075\n",
      "Acc: 0.7392541282701466\n",
      "TP: 3440\tFP: 1680\tTN: 8513\tFN: 2536\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233224905608022\n",
      "Acc: 0.53\n",
      "TP: 3\tFP: 0\tTN: 50\tFN: 47\n",
      "<2019-06-18 02:18:27.144227>  Epoch 15 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4879 - acc: 0.7556\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.901247562506422\n",
      "Acc: 0.7422846187148247\n",
      "TP: 3298\tFP: 1489\tTN: 8704\tFN: 2678\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887845137633217\n",
      "Acc: 0.54\n",
      "TP: 5\tFP: 1\tTN: 49\tFN: 45\n",
      "<2019-06-18 02:18:34.107003>  Epoch 16 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4815 - acc: 0.7592\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.82008726811432\n",
      "Acc: 0.7446347949780444\n",
      "TP: 3577\tFP: 1730\tTN: 8463\tFN: 2399\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233232901582323\n",
      "Acc: 0.53\n",
      "TP: 4\tFP: 1\tTN: 49\tFN: 46\n",
      "<2019-06-18 02:18:41.131341>  Epoch 17 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4721 - acc: 0.7688\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.749602137668946\n",
      "Acc: 0.7466757375224194\n",
      "TP: 3742\tFP: 1862\tTN: 8331\tFN: 2234\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.57862066553143\n",
      "Acc: 0.52\n",
      "TP: 3\tFP: 1\tTN: 49\tFN: 47\n",
      "<2019-06-18 02:18:48.128910>  Epoch 18 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4659 - acc: 0.7698\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.683371623842081\n",
      "Acc: 0.7485929865792565\n",
      "TP: 3549\tFP: 1638\tTN: 8555\tFN: 2427\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54245737368411\n",
      "Acc: 0.55\n",
      "TP: 6\tFP: 1\tTN: 49\tFN: 44\n",
      "<2019-06-18 02:18:55.144348>  Epoch 19 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4608 - acc: 0.7742\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.621413381175289\n",
      "Acc: 0.7503865421485558\n",
      "TP: 3355\tFP: 1415\tTN: 8778\tFN: 2621\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:19:02.122053>  Epoch 20 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.4527 - acc: 0.7789\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.501790134860672\n",
      "Acc: 0.7538499597996166\n",
      "TP: 3390\tFP: 1394\tTN: 8799\tFN: 2586\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887845137633217\n",
      "Acc: 0.54\n",
      "TP: 5\tFP: 1\tTN: 49\tFN: 45\n",
      "<2019-06-18 02:19:09.034627>  Epoch 21 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.4466 - acc: 0.7816\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.501788849095743\n",
      "Acc: 0.7538499597996166\n",
      "TP: 3364\tFP: 1368\tTN: 8825\tFN: 2612\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887845137633217\n",
      "Acc: 0.54\n",
      "TP: 5\tFP: 1\tTN: 49\tFN: 45\n",
      "<2019-06-18 02:19:15.951783>  Epoch 22 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.4392 - acc: 0.7871\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.456954011102434\n",
      "Acc: 0.7551487414187643\n",
      "TP: 3860\tFP: 1843\tTN: 8350\tFN: 2116\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233224905608022\n",
      "Acc: 0.53\n",
      "TP: 3\tFP: 0\tTN: 50\tFN: 47\n",
      "<2019-06-18 02:19:22.869290>  Epoch 23 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.4322 - acc: 0.7911\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.488994536501826\n",
      "Acc: 0.7542210402622302\n",
      "TP: 3822\tFP: 1820\tTN: 8373\tFN: 2154\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233224905608022\n",
      "Acc: 0.53\n",
      "TP: 3\tFP: 0\tTN: 50\tFN: 47\n",
      "<2019-06-18 02:19:29.787092>  Epoch 24 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.4251 - acc: 0.7960\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.441997971261085\n",
      "Acc: 0.7555816686251469\n",
      "TP: 3801\tFP: 1777\tTN: 8416\tFN: 2175\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887845137633217\n",
      "Acc: 0.54\n",
      "TP: 5\tFP: 1\tTN: 49\tFN: 45\n",
      "<2019-06-18 02:19:36.715909>  Epoch 25 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 91us/step - loss: 0.4208 - acc: 0.7978\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.362952028623921\n",
      "Acc: 0.7578699981445977\n",
      "TP: 3639\tFP: 1578\tTN: 8615\tFN: 2337\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233224905608022\n",
      "Acc: 0.53\n",
      "TP: 3\tFP: 0\tTN: 50\tFN: 47\n",
      "<2019-06-18 02:19:43.646328>  Epoch 26 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4135 - acc: 0.8015\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.337336946349991\n",
      "Acc: 0.7586121590698249\n",
      "TP: 4020\tFP: 1947\tTN: 8246\tFN: 1956\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887845137633217\n",
      "Acc: 0.54\n",
      "TP: 5\tFP: 1\tTN: 49\tFN: 45\n",
      "<2019-06-18 02:19:50.607811>  Epoch 27 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4087 - acc: 0.8054\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.491106712346882\n",
      "Acc: 0.7541591935184613\n",
      "TP: 3337\tFP: 1336\tTN: 8857\tFN: 2639\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887845137633217\n",
      "Acc: 0.54\n",
      "TP: 5\tFP: 1\tTN: 49\tFN: 45\n",
      "<2019-06-18 02:19:57.532110>  Epoch 28 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4011 - acc: 0.8104\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.358660273179877\n",
      "Acc: 0.7579936916321356\n",
      "TP: 3246\tFP: 1183\tTN: 9010\tFN: 2730\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887837141658915\n",
      "Acc: 0.54\n",
      "TP: 4\tFP: 0\tTN: 50\tFN: 46\n",
      "<2019-06-18 02:20:04.469987>  Epoch 29 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3907 - acc: 0.8177\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.3565333604906\n",
      "Acc: 0.7580555383759046\n",
      "TP: 3433\tFP: 1369\tTN: 8824\tFN: 2543\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197061613760702\n",
      "Acc: 0.56\n",
      "TP: 6\tFP: 0\tTN: 50\tFN: 44\n",
      "<2019-06-18 02:20:11.417866>  Epoch 30 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3882 - acc: 0.8189\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.625735797167712\n",
      "Acc: 0.750262848661018\n",
      "TP: 4368\tFP: 2430\tTN: 7763\tFN: 1608\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542449377709808\n",
      "Acc: 0.55\n",
      "TP: 5\tFP: 0\tTN: 50\tFN: 45\n",
      "<2019-06-18 02:20:18.370333>  Epoch 31 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3840 - acc: 0.8191\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.450569119024829\n",
      "Acc: 0.7553342816500711\n",
      "TP: 4337\tFP: 2317\tTN: 7876\tFN: 1639\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887837141658915\n",
      "Acc: 0.54\n",
      "TP: 4\tFP: 0\tTN: 50\tFN: 46\n",
      "<2019-06-18 02:20:25.307955>  Epoch 32 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3778 - acc: 0.8224\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.294612850079314\n",
      "Acc: 0.7598490939452038\n",
      "TP: 4002\tFP: 1909\tTN: 8284\tFN: 1974\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887837141658915\n",
      "Acc: 0.54\n",
      "TP: 4\tFP: 0\tTN: 50\tFN: 46\n",
      "<2019-06-18 02:20:32.303120>  Epoch 33 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3685 - acc: 0.8285\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.271110240365422\n",
      "Acc: 0.7605294081266621\n",
      "TP: 3904\tFP: 1800\tTN: 8393\tFN: 2072\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542449377709808\n",
      "Acc: 0.55\n",
      "TP: 5\tFP: 0\tTN: 50\tFN: 45\n",
      "<2019-06-18 02:20:39.275665>  Epoch 34 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3617 - acc: 0.8328\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.476202300912801\n",
      "Acc: 0.7545921207248438\n",
      "TP: 4322\tFP: 2314\tTN: 7879\tFN: 1654\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542449377709808\n",
      "Acc: 0.55\n",
      "TP: 5\tFP: 0\tTN: 50\tFN: 45\n",
      "<2019-06-18 02:20:46.184783>  Epoch 35 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3627 - acc: 0.8312\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.249759417946992\n",
      "Acc: 0.7611478755643515\n",
      "TP: 4122\tFP: 2008\tTN: 8185\tFN: 1854\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542473365632713\n",
      "Acc: 0.55\n",
      "TP: 8\tFP: 3\tTN: 47\tFN: 42\n",
      "<2019-06-18 02:20:53.140179>  Epoch 36 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3511 - acc: 0.8378\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.309574032980391\n",
      "Acc: 0.7594161667388212\n",
      "TP: 4165\tFP: 2079\tTN: 8114\tFN: 1811\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197093597657908\n",
      "Acc: 0.56\n",
      "TP: 10\tFP: 4\tTN: 46\tFN: 40\n",
      "<2019-06-18 02:21:00.067083>  Epoch 37 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3466 - acc: 0.8414\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.234794031583636\n",
      "Acc: 0.7615808027707341\n",
      "TP: 3874\tFP: 1753\tTN: 8440\tFN: 2102\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233232901582323\n",
      "Acc: 0.53\n",
      "TP: 4\tFP: 1\tTN: 49\tFN: 46\n",
      "<2019-06-18 02:21:07.069443>  Epoch 38 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3367 - acc: 0.8476\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.288202044893076\n",
      "Acc: 0.7600346341765106\n",
      "TP: 3955\tFP: 1859\tTN: 8334\tFN: 2021\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233232901582323\n",
      "Acc: 0.53\n",
      "TP: 4\tFP: 1\tTN: 49\tFN: 46\n",
      "<2019-06-18 02:21:14.018843>  Epoch 39 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3314 - acc: 0.8475\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.241206419249787\n",
      "Acc: 0.7613952625394274\n",
      "TP: 3953\tFP: 1835\tTN: 8358\tFN: 2023\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 02:21:20.989104>  Epoch 40 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3230 - acc: 0.8535\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.22410615834508\n",
      "Acc: 0.7618900364895789\n",
      "TP: 3731\tFP: 1605\tTN: 8588\tFN: 2245\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.81553454588718\n",
      "Acc: 0.6\n",
      "TP: 13\tFP: 3\tTN: 47\tFN: 37\n",
      "<2019-06-18 02:21:27.963282>  Epoch 41 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3210 - acc: 0.8537\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.258298718302417\n",
      "Acc: 0.7609004885892757\n",
      "TP: 4014\tFP: 1904\tTN: 8289\tFN: 1962\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542473365632713\n",
      "Acc: 0.55\n",
      "TP: 8\tFP: 3\tTN: 47\tFN: 42\n",
      "<2019-06-18 02:21:34.915155>  Epoch 42 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3116 - acc: 0.8586\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.204879726538556\n",
      "Acc: 0.7624466571834992\n",
      "TP: 3711\tFP: 1576\tTN: 8617\tFN: 2265\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 02:21:41.869592>  Epoch 43 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3072 - acc: 0.8616\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.41638862547685\n",
      "Acc: 0.7563238295503741\n",
      "TP: 4298\tFP: 2262\tTN: 7931\tFN: 1678\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197069609735001\n",
      "Acc: 0.56\n",
      "TP: 7\tFP: 1\tTN: 49\tFN: 43\n",
      "<2019-06-18 02:21:48.801739>  Epoch 44 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3046 - acc: 0.8630\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.412117491724217\n",
      "Acc: 0.7564475230379121\n",
      "TP: 4322\tFP: 2284\tTN: 7909\tFN: 1654\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887853133607518\n",
      "Acc: 0.54\n",
      "TP: 6\tFP: 2\tTN: 48\tFN: 44\n",
      "<2019-06-18 02:21:55.775844>  Epoch 45 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2980 - acc: 0.8681\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.315964216475217\n",
      "Acc: 0.7592306265075144\n",
      "TP: 3795\tFP: 1712\tTN: 8481\tFN: 2181\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:22:02.738881>  Epoch 46 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2903 - acc: 0.8702\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.281796481671565\n",
      "Acc: 0.7602201744078174\n",
      "TP: 4014\tFP: 1915\tTN: 8278\tFN: 1962\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233240897556627\n",
      "Acc: 0.53\n",
      "TP: 5\tFP: 2\tTN: 48\tFN: 45\n",
      "<2019-06-18 02:22:09.732136>  Epoch 47 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2857 - acc: 0.8731\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.313830726603788\n",
      "Acc: 0.7592924732512834\n",
      "TP: 3849\tFP: 1765\tTN: 8428\tFN: 2127\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54245737368411\n",
      "Acc: 0.55\n",
      "TP: 6\tFP: 1\tTN: 49\tFN: 44\n",
      "<2019-06-18 02:22:16.638122>  Epoch 48 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2786 - acc: 0.8770\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.324531309134173\n",
      "Acc: 0.7589832395324386\n",
      "TP: 4249\tFP: 2170\tTN: 8023\tFN: 1727\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 02:22:23.539540>  Epoch 49 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2766 - acc: 0.8779\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.25830054804482\n",
      "Acc: 0.7609004885892757\n",
      "TP: 4051\tFP: 1941\tTN: 8252\tFN: 1925\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887853133607518\n",
      "Acc: 0.54\n",
      "TP: 6\tFP: 2\tTN: 48\tFN: 44\n",
      "<2019-06-18 02:22:30.497913>  Epoch 50 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2687 - acc: 0.8822\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.35870087368023\n",
      "Acc: 0.7579936916321356\n",
      "TP: 4067\tFP: 2004\tTN: 8189\tFN: 1909\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233272881453832\n",
      "Acc: 0.53\n",
      "TP: 9\tFP: 6\tTN: 44\tFN: 41\n",
      "<2019-06-18 02:22:37.408874>  Epoch 51 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2636 - acc: 0.8846\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.151485263213397\n",
      "Acc: 0.7639928257777228\n",
      "TP: 3904\tFP: 1744\tTN: 8449\tFN: 2072\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887869125556124\n",
      "Acc: 0.54\n",
      "TP: 8\tFP: 4\tTN: 46\tFN: 42\n",
      "<2019-06-18 02:22:44.378429>  Epoch 52 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2581 - acc: 0.8880\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.390751487389087\n",
      "Acc: 0.7570659904756014\n",
      "TP: 4233\tFP: 2185\tTN: 8008\tFN: 1743\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197093597657908\n",
      "Acc: 0.56\n",
      "TP: 10\tFP: 4\tTN: 46\tFN: 40\n",
      "<2019-06-18 02:22:51.306622>  Epoch 53 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2567 - acc: 0.8879\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.172836876871788\n",
      "Acc: 0.7633743583400334\n",
      "TP: 3702\tFP: 1552\tTN: 8641\tFN: 2274\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542465369658412\n",
      "Acc: 0.55\n",
      "TP: 7\tFP: 2\tTN: 48\tFN: 43\n",
      "<2019-06-18 02:22:58.235882>  Epoch 54 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2492 - acc: 0.8914\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.207023403624445\n",
      "Acc: 0.7623848104397304\n",
      "TP: 3863\tFP: 1729\tTN: 8464\tFN: 2113\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887853133607518\n",
      "Acc: 0.54\n",
      "TP: 6\tFP: 2\tTN: 48\tFN: 44\n",
      "<2019-06-18 02:23:05.162842>  Epoch 55 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2474 - acc: 0.8928\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.315940330918979\n",
      "Acc: 0.7592306265075144\n",
      "TP: 3312\tFP: 1229\tTN: 8964\tFN: 2664\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 02:23:12.083804>  Epoch 56 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2399 - acc: 0.8952\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.401426898598304\n",
      "Acc: 0.7567567567567568\n",
      "TP: 4124\tFP: 2081\tTN: 8112\tFN: 1852\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 02:23:19.056425>  Epoch 57 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2385 - acc: 0.8970\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.350157023695042\n",
      "Acc: 0.7582410786072113\n",
      "TP: 4083\tFP: 2016\tTN: 8177\tFN: 1893\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887893113479027\n",
      "Acc: 0.54\n",
      "TP: 11\tFP: 7\tTN: 43\tFN: 39\n",
      "<2019-06-18 02:23:25.998336>  Epoch 58 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2307 - acc: 0.9007\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.2454723110377\n",
      "Acc: 0.7612715690518894\n",
      "TP: 3823\tFP: 1707\tTN: 8486\tFN: 2153\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197093597657908\n",
      "Acc: 0.56\n",
      "TP: 10\tFP: 4\tTN: 46\tFN: 40\n",
      "<2019-06-18 02:23:32.975742>  Epoch 59 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2266 - acc: 0.9019\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.254025458092395\n",
      "Acc: 0.7610241820768137\n",
      "TP: 3995\tFP: 1883\tTN: 8310\tFN: 1981\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197093597657908\n",
      "Acc: 0.56\n",
      "TP: 10\tFP: 4\tTN: 46\tFN: 40\n",
      "<2019-06-18 02:23:39.914992>  Epoch 60 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2239 - acc: 0.9027\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.296754944685285\n",
      "Acc: 0.7597872472014349\n",
      "TP: 4122\tFP: 2030\tTN: 8163\tFN: 1854\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.88786112958182\n",
      "Acc: 0.54\n",
      "TP: 7\tFP: 3\tTN: 47\tFN: 43\n",
      "<2019-06-18 02:23:46.823298>  Epoch 61 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2182 - acc: 0.9065\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.27112240567978\n",
      "Acc: 0.7605294081266621\n",
      "TP: 4150\tFP: 2046\tTN: 8147\tFN: 1826\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233256889505228\n",
      "Acc: 0.53\n",
      "TP: 7\tFP: 4\tTN: 46\tFN: 43\n",
      "<2019-06-18 02:23:53.783648>  Epoch 62 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2102 - acc: 0.9108\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.294614185296743\n",
      "Acc: 0.7598490939452038\n",
      "TP: 4029\tFP: 1936\tTN: 8257\tFN: 1947\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 02:24:00.702868>  Epoch 63 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2120 - acc: 0.9091\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.194206639596718\n",
      "Acc: 0.762755890902344\n",
      "TP: 3867\tFP: 1727\tTN: 8466\tFN: 2109\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542473365632713\n",
      "Acc: 0.55\n",
      "TP: 8\tFP: 3\tTN: 47\tFN: 42\n",
      "<2019-06-18 02:24:07.648844>  Epoch 64 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2026 - acc: 0.9148\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.36082180261732\n",
      "Acc: 0.7579318448883666\n",
      "TP: 3759\tFP: 1697\tTN: 8496\tFN: 2217\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197117585580813\n",
      "Acc: 0.56\n",
      "TP: 13\tFP: 7\tTN: 43\tFN: 37\n",
      "<2019-06-18 02:24:14.592966>  Epoch 65 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2018 - acc: 0.9148\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.142936467978473\n",
      "Acc: 0.7642402127527985\n",
      "TP: 3820\tFP: 1656\tTN: 8537\tFN: 2156\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54250534952992\n",
      "Acc: 0.55\n",
      "TP: 12\tFP: 7\tTN: 43\tFN: 38\n",
      "<2019-06-18 02:24:21.579855>  Epoch 66 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1968 - acc: 0.9167\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.294617350256576\n",
      "Acc: 0.7598490939452038\n",
      "TP: 4093\tFP: 2000\tTN: 8193\tFN: 1883\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506350053656902\n",
      "Acc: 0.58\n",
      "TP: 16\tFP: 8\tTN: 42\tFN: 34\n",
      "<2019-06-18 02:24:28.506377>  Epoch 67 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1922 - acc: 0.9196\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.360820764114877\n",
      "Acc: 0.7579318448883666\n",
      "TP: 3738\tFP: 1676\tTN: 8517\tFN: 2238\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.19710159363221\n",
      "Acc: 0.56\n",
      "TP: 11\tFP: 5\tTN: 45\tFN: 39\n",
      "<2019-06-18 02:24:35.427947>  Epoch 68 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1927 - acc: 0.9197\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.37151793442294\n",
      "Acc: 0.7576226111695219\n",
      "TP: 4069\tFP: 2012\tTN: 8181\tFN: 1907\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887901109453328\n",
      "Acc: 0.54\n",
      "TP: 12\tFP: 8\tTN: 42\tFN: 38\n",
      "<2019-06-18 02:24:42.337872>  Epoch 69 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1863 - acc: 0.9208\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.46979421561392\n",
      "Acc: 0.7547776609561506\n",
      "TP: 4330\tFP: 2319\tTN: 7874\tFN: 1646\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197109589606512\n",
      "Acc: 0.56\n",
      "TP: 12\tFP: 6\tTN: 44\tFN: 38\n",
      "<2019-06-18 02:24:49.255841>  Epoch 70 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1856 - acc: 0.9226\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.326662969263197\n",
      "Acc: 0.7589213927886697\n",
      "TP: 4158\tFP: 2080\tTN: 8113\tFN: 1818\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.88790910542763\n",
      "Acc: 0.54\n",
      "TP: 13\tFP: 9\tTN: 41\tFN: 37\n",
      "<2019-06-18 02:24:56.210815>  Epoch 71 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1821 - acc: 0.9233\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.480468934488174\n",
      "Acc: 0.754468427237306\n",
      "TP: 4207\tFP: 2201\tTN: 7992\tFN: 1769\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815550537835783\n",
      "Acc: 0.6\n",
      "TP: 15\tFP: 5\tTN: 45\tFN: 35\n",
      "<2019-06-18 02:25:03.131107>  Epoch 72 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1738 - acc: 0.9282\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.243350640313148\n",
      "Acc: 0.7613334157956584\n",
      "TP: 4116\tFP: 1999\tTN: 8194\tFN: 1860\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160938301784888\n",
      "Acc: 0.59\n",
      "TP: 14\tFP: 5\tTN: 45\tFN: 36\n",
      "<2019-06-18 02:25:10.078700>  Epoch 73 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1720 - acc: 0.9279\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.309558653253704\n",
      "Acc: 0.7594161667388212\n",
      "TP: 3854\tFP: 1768\tTN: 8425\tFN: 2122\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851737817606008\n",
      "Acc: 0.57\n",
      "TP: 15\tFP: 8\tTN: 42\tFN: 35\n",
      "<2019-06-18 02:25:17.010759>  Epoch 74 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1695 - acc: 0.9299\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.43987788301645\n",
      "Acc: 0.7556435153689158\n",
      "TP: 4126\tFP: 2101\tTN: 8092\tFN: 1850\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233288873402433\n",
      "Acc: 0.53\n",
      "TP: 11\tFP: 8\tTN: 42\tFN: 39\n",
      "<2019-06-18 02:25:23.954897>  Epoch 75 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1666 - acc: 0.9318\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.429198812322426\n",
      "Acc: 0.7559527490877606\n",
      "TP: 4161\tFP: 2131\tTN: 8062\tFN: 1815\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197141573503721\n",
      "Acc: 0.56\n",
      "TP: 16\tFP: 10\tTN: 40\tFN: 34\n",
      "<2019-06-18 02:25:30.916345>  Epoch 76 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1652 - acc: 0.9321\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.427064877378522\n",
      "Acc: 0.7560145958315295\n",
      "TP: 4206\tFP: 2175\tTN: 8018\tFN: 1770\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924080393249252\n",
      "Acc: 0.51\n",
      "TP: 11\tFP: 10\tTN: 40\tFN: 39\n",
      "<2019-06-18 02:25:37.855644>  Epoch 77 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1614 - acc: 0.9336\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.330928366526136\n",
      "Acc: 0.7587976993011318\n",
      "TP: 4018\tFP: 1942\tTN: 8251\tFN: 1958\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269452165249753\n",
      "Acc: 0.5\n",
      "TP: 8\tFP: 8\tTN: 42\tFN: 42\n",
      "<2019-06-18 02:25:44.794236>  Epoch 78 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1567 - acc: 0.9364\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.271114048207721\n",
      "Acc: 0.7605294081266621\n",
      "TP: 3981\tFP: 1877\tTN: 8316\tFN: 1995\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578660645402937\n",
      "Acc: 0.52\n",
      "TP: 8\tFP: 6\tTN: 44\tFN: 42\n",
      "<2019-06-18 02:25:51.742833>  Epoch 79 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1585 - acc: 0.9351\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.427056569358962\n",
      "Acc: 0.7560145958315295\n",
      "TP: 4038\tFP: 2007\tTN: 8186\tFN: 1938\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269460161224057\n",
      "Acc: 0.5\n",
      "TP: 9\tFP: 9\tTN: 41\tFN: 41\n",
      "<2019-06-18 02:25:58.702193>  Epoch 80 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1513 - acc: 0.9379\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.228395787331735\n",
      "Acc: 0.7617663430020409\n",
      "TP: 4081\tFP: 1957\tTN: 8236\tFN: 1895\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233256889505228\n",
      "Acc: 0.53\n",
      "TP: 7\tFP: 4\tTN: 46\tFN: 43\n",
      "<2019-06-18 02:26:05.655783>  Epoch 81 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1520 - acc: 0.9379\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.471924787788002\n",
      "Acc: 0.7547158142123818\n",
      "TP: 4217\tFP: 2207\tTN: 7986\tFN: 1759\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887901109453328\n",
      "Acc: 0.54\n",
      "TP: 12\tFP: 8\tTN: 42\tFN: 38\n",
      "<2019-06-18 02:26:12.593526>  Epoch 82 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1503 - acc: 0.9387\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.427046777764478\n",
      "Acc: 0.7560145958315295\n",
      "TP: 3840\tFP: 1809\tTN: 8384\tFN: 2136\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851753809554612\n",
      "Acc: 0.57\n",
      "TP: 17\tFP: 10\tTN: 40\tFN: 33\n",
      "<2019-06-18 02:26:19.563849>  Epoch 83 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1484 - acc: 0.9402\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.382173317370716\n",
      "Acc: 0.7573133774506772\n",
      "TP: 3555\tFP: 1503\tTN: 8690\tFN: 2421\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233304865351037\n",
      "Acc: 0.53\n",
      "TP: 13\tFP: 10\tTN: 40\tFN: 37\n",
      "<2019-06-18 02:26:26.491438>  Epoch 84 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1434 - acc: 0.9415\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.196334442430947\n",
      "Acc: 0.762694044158575\n",
      "TP: 3698\tFP: 1559\tTN: 8634\tFN: 2278\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542521341478523\n",
      "Acc: 0.55\n",
      "TP: 14\tFP: 9\tTN: 41\tFN: 36\n",
      "<2019-06-18 02:26:33.457714>  Epoch 85 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1411 - acc: 0.9432\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.196331228018614\n",
      "Acc: 0.762694044158575\n",
      "TP: 3633\tFP: 1494\tTN: 8699\tFN: 2343\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54250534952992\n",
      "Acc: 0.55\n",
      "TP: 12\tFP: 7\tTN: 43\tFN: 38\n",
      "<2019-06-18 02:26:40.407264>  Epoch 86 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1400 - acc: 0.9438\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.254010721248175\n",
      "Acc: 0.7610241820768137\n",
      "TP: 3697\tFP: 1585\tTN: 8608\tFN: 2279\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578684633325842\n",
      "Acc: 0.52\n",
      "TP: 11\tFP: 9\tTN: 41\tFN: 39\n",
      "<2019-06-18 02:26:47.331079>  Epoch 87 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1370 - acc: 0.9453\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.429201631114777\n",
      "Acc: 0.7559527490877606\n",
      "TP: 4218\tFP: 2188\tTN: 8005\tFN: 1758\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542513345504222\n",
      "Acc: 0.55\n",
      "TP: 13\tFP: 8\tTN: 42\tFN: 37\n",
      "<2019-06-18 02:26:54.277329>  Epoch 88 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1358 - acc: 0.9448\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.343751707736017\n",
      "Acc: 0.7584266188385181\n",
      "TP: 4147\tFP: 2077\tTN: 8116\tFN: 1829\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124830981757682\n",
      "Acc: 0.62\n",
      "TP: 24\tFP: 12\tTN: 38\tFN: 26\n",
      "<2019-06-18 02:27:01.259276>  Epoch 89 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1322 - acc: 0.9471\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.262578852409577\n",
      "Acc: 0.7607767951017379\n",
      "TP: 4172\tFP: 2064\tTN: 8129\tFN: 1804\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54254532940143\n",
      "Acc: 0.55\n",
      "TP: 17\tFP: 12\tTN: 38\tFN: 33\n",
      "<2019-06-18 02:27:08.270431>  Epoch 90 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1301 - acc: 0.9472\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.36724234994554\n",
      "Acc: 0.7577463046570598\n",
      "TP: 4003\tFP: 1944\tTN: 8249\tFN: 1973\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887925097376232\n",
      "Acc: 0.54\n",
      "TP: 15\tFP: 11\tTN: 39\tFN: 35\n",
      "<2019-06-18 02:27:15.223152>  Epoch 91 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1283 - acc: 0.9482\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.34588430746249\n",
      "Acc: 0.7583647720947492\n",
      "TP: 4075\tFP: 2006\tTN: 8187\tFN: 1901\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.470170769860978\n",
      "Acc: 0.61\n",
      "TP: 17\tFP: 6\tTN: 44\tFN: 33\n",
      "<2019-06-18 02:27:22.197580>  Epoch 92 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1255 - acc: 0.9498\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.30956607112831\n",
      "Acc: 0.7594161667388212\n",
      "TP: 4004\tFP: 1918\tTN: 8275\tFN: 1972\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542497353555618\n",
      "Acc: 0.55\n",
      "TP: 11\tFP: 6\tTN: 44\tFN: 39\n",
      "<2019-06-18 02:27:29.152222>  Epoch 93 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1257 - acc: 0.9489\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.362965430250712\n",
      "Acc: 0.7578699981445977\n",
      "TP: 3910\tFP: 1849\tTN: 8344\tFN: 2066\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197157565452324\n",
      "Acc: 0.56\n",
      "TP: 18\tFP: 12\tTN: 38\tFN: 32\n",
      "<2019-06-18 02:27:36.081472>  Epoch 94 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1205 - acc: 0.9524\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.307434460451786\n",
      "Acc: 0.7594780134825901\n",
      "TP: 4096\tFP: 2009\tTN: 8184\tFN: 1880\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160946297759192\n",
      "Acc: 0.59\n",
      "TP: 15\tFP: 6\tTN: 44\tFN: 35\n",
      "<2019-06-18 02:27:43.042559>  Epoch 95 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1187 - acc: 0.9532\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.431325873369193\n",
      "Acc: 0.7558909023439916\n",
      "TP: 3977\tFP: 1948\tTN: 8245\tFN: 1999\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851769801503215\n",
      "Acc: 0.57\n",
      "TP: 19\tFP: 12\tTN: 38\tFN: 31\n",
      "<2019-06-18 02:27:49.954465>  Epoch 96 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1183 - acc: 0.9536\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.382195224827058\n",
      "Acc: 0.7573133774506772\n",
      "TP: 3998\tFP: 1946\tTN: 8247\tFN: 1978\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.1609862776307\n",
      "Acc: 0.59\n",
      "TP: 20\tFP: 11\tTN: 39\tFN: 30\n",
      "<2019-06-18 02:27:56.926263>  Epoch 97 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1167 - acc: 0.9536\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.305305371852622\n",
      "Acc: 0.7595398602263591\n",
      "TP: 4239\tFP: 2151\tTN: 8042\tFN: 1737\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506366045605505\n",
      "Acc: 0.58\n",
      "TP: 18\tFP: 10\tTN: 40\tFN: 32\n",
      "<2019-06-18 02:28:03.891323>  Epoch 98 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1142 - acc: 0.9549\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.403565927149437\n",
      "Acc: 0.7566949100129878\n",
      "TP: 4182\tFP: 2140\tTN: 8053\tFN: 1794\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.470194757783883\n",
      "Acc: 0.61\n",
      "TP: 20\tFP: 9\tTN: 41\tFN: 30\n",
      "<2019-06-18 02:28:10.791342>  Epoch 99 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1140 - acc: 0.9541\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.345882873340067\n",
      "Acc: 0.7583647720947492\n",
      "TP: 4046\tFP: 1977\tTN: 8216\tFN: 1930\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.12482298578338\n",
      "Acc: 0.62\n",
      "TP: 23\tFP: 11\tTN: 39\tFN: 27\n",
      "<2019-06-18 02:28:17.760335>  Epoch 100 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1160 - acc: 0.9538\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.38646299580987\n",
      "Acc: 0.7571896839631393\n",
      "TP: 3906\tFP: 1856\tTN: 8337\tFN: 2070\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197125581555115\n",
      "Acc: 0.56\n",
      "TP: 14\tFP: 8\tTN: 42\tFN: 36\n",
      "<2019-06-18 02:28:24.667666>  Epoch 101 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1081 - acc: 0.9569\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.32237734592883\n",
      "Acc: 0.7590450862762076\n",
      "TP: 3889\tFP: 1809\tTN: 8384\tFN: 2087\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815566529784387\n",
      "Acc: 0.6\n",
      "TP: 17\tFP: 7\tTN: 43\tFN: 33\n",
      "<2019-06-18 02:28:31.612892>  Epoch 102 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1068 - acc: 0.9586\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.311701687457129\n",
      "Acc: 0.7593543199950522\n",
      "TP: 3993\tFP: 1908\tTN: 8285\tFN: 1983\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.5063420576826\n",
      "Acc: 0.58\n",
      "TP: 15\tFP: 7\tTN: 43\tFN: 35\n",
      "<2019-06-18 02:28:38.573243>  Epoch 103 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1072 - acc: 0.9590\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.442012015770342\n",
      "Acc: 0.7555816686251469\n",
      "TP: 4085\tFP: 2061\tTN: 8132\tFN: 1891\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815598513681593\n",
      "Acc: 0.6\n",
      "TP: 21\tFP: 11\tTN: 39\tFN: 29\n",
      "<2019-06-18 02:28:45.478065>  Epoch 104 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1049 - acc: 0.9600\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.309560581901103\n",
      "Acc: 0.7594161667388212\n",
      "TP: 3893\tFP: 1807\tTN: 8386\tFN: 2083\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.85174581358031\n",
      "Acc: 0.57\n",
      "TP: 16\tFP: 9\tTN: 41\tFN: 34\n",
      "<2019-06-18 02:28:52.436834>  Epoch 105 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1064 - acc: 0.9583\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.478334900639274\n",
      "Acc: 0.7545302739810749\n",
      "TP: 4250\tFP: 2243\tTN: 7950\tFN: 1726\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506326065733997\n",
      "Acc: 0.58\n",
      "TP: 13\tFP: 5\tTN: 45\tFN: 37\n",
      "<2019-06-18 02:28:59.380187>  Epoch 106 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1038 - acc: 0.9589\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.493283522606015\n",
      "Acc: 0.7540973467746923\n",
      "TP: 4159\tFP: 2159\tTN: 8034\tFN: 1817\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.19710159363221\n",
      "Acc: 0.56\n",
      "TP: 11\tFP: 5\tTN: 45\tFN: 39\n",
      "<2019-06-18 02:29:06.348513>  Epoch 107 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1020 - acc: 0.9601\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.277526930398846\n",
      "Acc: 0.7603438678953554\n",
      "TP: 4070\tFP: 1969\tTN: 8224\tFN: 1906\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815550537835783\n",
      "Acc: 0.6\n",
      "TP: 15\tFP: 5\tTN: 45\tFN: 35\n",
      "<2019-06-18 02:29:13.330614>  Epoch 108 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0995 - acc: 0.9610\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.369373812264575\n",
      "Acc: 0.7576844579132909\n",
      "TP: 3908\tFP: 1850\tTN: 8343\tFN: 2068\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815606509655895\n",
      "Acc: 0.6\n",
      "TP: 22\tFP: 12\tTN: 38\tFN: 28\n",
      "<2019-06-18 02:29:20.238090>  Epoch 109 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0959 - acc: 0.9627\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.354430481715054\n",
      "Acc: 0.7581173851196734\n",
      "TP: 4106\tFP: 2041\tTN: 8152\tFN: 1870\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506382037554108\n",
      "Acc: 0.58\n",
      "TP: 20\tFP: 12\tTN: 38\tFN: 30\n",
      "<2019-06-18 02:29:27.189743>  Epoch 110 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0970 - acc: 0.9625\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.283931999095385\n",
      "Acc: 0.7601583276640484\n",
      "TP: 4001\tFP: 1903\tTN: 8290\tFN: 1975\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851761805528913\n",
      "Acc: 0.57\n",
      "TP: 18\tFP: 11\tTN: 39\tFN: 32\n",
      "<2019-06-18 02:29:34.177362>  Epoch 111 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0970 - acc: 0.9615\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.311699857714723\n",
      "Acc: 0.7593543199950522\n",
      "TP: 3956\tFP: 1871\tTN: 8322\tFN: 2020\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506374041579807\n",
      "Acc: 0.58\n",
      "TP: 19\tFP: 11\tTN: 39\tFN: 31\n",
      "<2019-06-18 02:29:41.095420>  Epoch 112 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0912 - acc: 0.9650\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.305274167326774\n",
      "Acc: 0.7595398602263591\n",
      "TP: 3608\tFP: 1520\tTN: 8673\tFN: 2368\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851737817606008\n",
      "Acc: 0.57\n",
      "TP: 15\tFP: 8\tTN: 42\tFN: 35\n",
      "<2019-06-18 02:29:48.103277>  Epoch 113 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0925 - acc: 0.9636\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.563789868172767\n",
      "Acc: 0.7520564042303173\n",
      "TP: 4423\tFP: 2456\tTN: 7737\tFN: 1553\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160978281656398\n",
      "Acc: 0.59\n",
      "TP: 19\tFP: 10\tTN: 40\tFN: 31\n",
      "<2019-06-18 02:29:55.045473>  Epoch 114 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0939 - acc: 0.9632\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.32239188496306\n",
      "Acc: 0.7590450862762076\n",
      "TP: 4183\tFP: 2103\tTN: 8090\tFN: 1793\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.85174581358031\n",
      "Acc: 0.57\n",
      "TP: 16\tFP: 9\tTN: 41\tFN: 34\n",
      "<2019-06-18 02:30:02.041274>  Epoch 115 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0944 - acc: 0.9632\n",
      "DEVELOPMENT SET (ENGLISH)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log Loss: 8.324515286525022\n",
      "Acc: 0.7589832395324386\n",
      "TP: 3925\tFP: 1846\tTN: 8347\tFN: 2051\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851777797477519\n",
      "Acc: 0.57\n",
      "TP: 20\tFP: 13\tTN: 37\tFN: 30\n",
      "<2019-06-18 02:30:08.969307>  Epoch 116 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0895 - acc: 0.9644\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.322379966911193\n",
      "Acc: 0.7590450862762076\n",
      "TP: 3942\tFP: 1862\tTN: 8331\tFN: 2034\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815606509655895\n",
      "Acc: 0.6\n",
      "TP: 22\tFP: 12\tTN: 38\tFN: 28\n",
      "<2019-06-18 02:30:15.899981>  Epoch 117 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0895 - acc: 0.9652\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.570195134679295\n",
      "Acc: 0.7518708639990105\n",
      "TP: 4358\tFP: 2394\tTN: 7799\tFN: 1618\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233288873402433\n",
      "Acc: 0.53\n",
      "TP: 11\tFP: 8\tTN: 42\tFN: 39\n",
      "<2019-06-18 02:30:22.828835>  Epoch 118 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0892 - acc: 0.9655\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.26470902896368\n",
      "Acc: 0.7607149483579689\n",
      "TP: 4051\tFP: 1944\tTN: 8249\tFN: 1925\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851729821631707\n",
      "Acc: 0.57\n",
      "TP: 14\tFP: 7\tTN: 43\tFN: 36\n",
      "<2019-06-18 02:30:29.770306>  Epoch 119 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0876 - acc: 0.9659\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.330935092065781\n",
      "Acc: 0.7587976993011318\n",
      "TP: 4154\tFP: 2078\tTN: 8115\tFN: 1822\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197133577529417\n",
      "Acc: 0.56\n",
      "TP: 15\tFP: 9\tTN: 41\tFN: 35\n",
      "<2019-06-18 02:30:36.714967>  Epoch 120 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0876 - acc: 0.9659\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.260437598496061\n",
      "Acc: 0.7608386418455069\n",
      "TP: 4069\tFP: 1960\tTN: 8233\tFN: 1907\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.19714956947802\n",
      "Acc: 0.56\n",
      "TP: 17\tFP: 11\tTN: 39\tFN: 33\n",
      "<2019-06-18 02:30:43.670419>  Epoch 121 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0866 - acc: 0.9664\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.301018858373302\n",
      "Acc: 0.7596635537138969\n",
      "TP: 3952\tFP: 1862\tTN: 8331\tFN: 2024\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160962289707795\n",
      "Acc: 0.59\n",
      "TP: 17\tFP: 8\tTN: 42\tFN: 33\n",
      "<2019-06-18 02:30:50.607305>  Epoch 122 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0868 - acc: 0.9671\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.294614234749242\n",
      "Acc: 0.7598490939452038\n",
      "TP: 4030\tFP: 1937\tTN: 8256\tFN: 1946\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197109589606512\n",
      "Acc: 0.56\n",
      "TP: 12\tFP: 6\tTN: 44\tFN: 38\n",
      "<2019-06-18 02:30:57.537642>  Epoch 123 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0830 - acc: 0.9679\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.365105596209284\n",
      "Acc: 0.7578081514008288\n",
      "TP: 3991\tFP: 1931\tTN: 8262\tFN: 1985\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197117585580813\n",
      "Acc: 0.56\n",
      "TP: 13\tFP: 7\tTN: 43\tFN: 37\n",
      "<2019-06-18 02:31:04.459604>  Epoch 124 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0810 - acc: 0.9689\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.35015687533755\n",
      "Acc: 0.7582410786072113\n",
      "TP: 4080\tFP: 2013\tTN: 8180\tFN: 1896\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506382037554111\n",
      "Acc: 0.58\n",
      "TP: 20\tFP: 12\tTN: 38\tFN: 30\n",
      "<2019-06-18 02:31:11.436386>  Epoch 125 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0800 - acc: 0.9698\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.463383410427683\n",
      "Acc: 0.7549632011874575\n",
      "TP: 4283\tFP: 2269\tTN: 7924\tFN: 1693\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160970285682097\n",
      "Acc: 0.59\n",
      "TP: 18\tFP: 9\tTN: 41\tFN: 32\n",
      "<2019-06-18 02:31:18.355383>  Epoch 126 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0807 - acc: 0.9693\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.200615862303037\n",
      "Acc: 0.7625703506710372\n",
      "TP: 3882\tFP: 1745\tTN: 8448\tFN: 2094\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54250534952992\n",
      "Acc: 0.55\n",
      "TP: 12\tFP: 7\tTN: 43\tFN: 38\n",
      "<2019-06-18 02:31:25.310234>  Epoch 127 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0785 - acc: 0.9703\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.350162315112263\n",
      "Acc: 0.7582410786072113\n",
      "TP: 4190\tFP: 2123\tTN: 8070\tFN: 1786\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506358049631203\n",
      "Acc: 0.58\n",
      "TP: 17\tFP: 9\tTN: 41\tFN: 33\n",
      "<2019-06-18 02:31:32.242771>  Epoch 128 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0797 - acc: 0.9697\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.476196119350627\n",
      "Acc: 0.7545921207248438\n",
      "TP: 4197\tFP: 2189\tTN: 8004\tFN: 1779\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887925097376232\n",
      "Acc: 0.54\n",
      "TP: 15\tFP: 11\tTN: 39\tFN: 35\n"
     ]
    }
   ],
   "source": [
    "model = build_model_concat_LSTM()\n",
    "\n",
    "bestEN = sys.float_info.max\n",
    "bestES = sys.float_info.max\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    printTrace('Epoch ' +str(epoch+1) + ' of ' +str(EPOCHS))\n",
    "    model.fit(train_set.X, train_set.y, batch_size=BATCH_SIZE, epochs=1, verbose=1)\n",
    "    print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "    \n",
    "    result_en = evaluate(model,dev_set)\n",
    "    \n",
    "    if result_en < bestEN:\n",
    "        bestEN = result_en\n",
    "        model.save(Save_path+'bestEN_perceptron.h5')\n",
    "    \n",
    "    print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "    result_es = evaluate(model,test_set)\n",
    "    if result_es < bestES:\n",
    "        bestEN = result_en\n",
    "        model.save(Save_path+'bestES_perceptron.h5')\n",
    "\n",
    "    \n",
    "model.save(Save_path+'final_perceptron.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ===== LAST: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.476196119350627\n",
      "Acc: 0.7545921207248438\n",
      "TP: 4197\tFP: 2189\tTN: 8004\tFN: 1779\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887925097376232\n",
      "Acc: 0.54\n",
      "TP: 15\tFP: 11\tTN: 39\tFN: 35\n",
      "\n",
      "\n",
      "\n",
      " ===== BEST IN DEV SET: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.476196119350627\n",
      "Acc: 0.7545921207248438\n",
      "TP: 4197\tFP: 2189\tTN: 8004\tFN: 1779\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887925097376232\n",
      "Acc: 0.54\n",
      "TP: 15\tFP: 11\tTN: 39\tFN: 35\n",
      "\n",
      "\n",
      "\n",
      " ===== BEST IN TEST SET: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.476196119350627\n",
      "Acc: 0.7545921207248438\n",
      "TP: 4197\tFP: 2189\tTN: 8004\tFN: 1779\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887925097376232\n",
      "Acc: 0.54\n",
      "TP: 15\tFP: 11\tTN: 39\tFN: 35\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "15.887925097376232"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\" ===== LAST: ====\")\n",
    "mode = load_model(Save_path+'final_perceptron.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\" ===== BEST IN DEV SET: ====\")\n",
    "mode = load_model(Save_path+'bestEN_perceptron.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\" ===== BEST IN TEST SET: ====\")\n",
    "mode = load_model(Save_path+'bestES_perceptron.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RUN CONCAT LSTM MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 25, 300)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            (None, 25, 300)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 25, 600)      0           input_3[0][0]                    \n",
      "                                                                 input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 256)          746496      concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            257         bidirectional_2[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 746,753\n",
      "Trainable params: 746,753\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "<2019-06-18 02:31:50.941636>  Epoch 1 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 7s 109us/step - loss: 0.6317 - acc: 0.6358\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 11.509398314482164\n",
      "Acc: 0.6667697445729482\n",
      "TP: 1256\tFP: 668\tTN: 9525\tFN: 4720\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:32:00.803236>  Epoch 2 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5920 - acc: 0.6803\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 11.364269523175322\n",
      "Acc: 0.6709753231492362\n",
      "TP: 3887\tFP: 3231\tTN: 6962\tFN: 2089\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:32:07.804246>  Epoch 3 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5814 - acc: 0.6886\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.304682185040129\n",
      "Acc: 0.7016513080586307\n",
      "TP: 2839\tFP: 1687\tTN: 8506\tFN: 3137\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 18.65097923312328\n",
      "Acc: 0.46\n",
      "TP: 1\tFP: 5\tTN: 45\tFN: 49\n",
      "<2019-06-18 02:32:14.789319>  Epoch 4 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5701 - acc: 0.6993\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.093205084057654\n",
      "Acc: 0.7077741356917558\n",
      "TP: 2895\tFP: 1644\tTN: 8549\tFN: 3081\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 18.99638298902099\n",
      "Acc: 0.45\n",
      "TP: 2\tFP: 7\tTN: 43\tFN: 48\n",
      "<2019-06-18 02:32:21.728362>  Epoch 5 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5629 - acc: 0.7044\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.266196188253838\n",
      "Acc: 0.7027645494464716\n",
      "TP: 2129\tFP: 959\tTN: 9234\tFN: 3847\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614799949327352\n",
      "Acc: 0.49\n",
      "TP: 2\tFP: 3\tTN: 47\tFN: 48\n",
      "<2019-06-18 02:32:28.667578>  Epoch 6 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5554 - acc: 0.7095\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.952228443794743\n",
      "Acc: 0.7118560207805059\n",
      "TP: 3096\tFP: 1779\tTN: 8414\tFN: 2880\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.96021969717367\n",
      "Acc: 0.48\n",
      "TP: 5\tFP: 7\tTN: 43\tFN: 45\n",
      "<2019-06-18 02:32:35.657438>  Epoch 7 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5498 - acc: 0.7134\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.83045143208477\n",
      "Acc: 0.7153812851753355\n",
      "TP: 2775\tFP: 1401\tTN: 8792\tFN: 3201\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.26942018135255\n",
      "Acc: 0.5\n",
      "TP: 4\tFP: 4\tTN: 46\tFN: 46\n",
      "<2019-06-18 02:32:42.618389>  Epoch 8 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5431 - acc: 0.7185\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.766399409901952\n",
      "Acc: 0.7172366874884037\n",
      "TP: 3438\tFP: 2034\tTN: 8159\tFN: 2538\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924056405326347\n",
      "Acc: 0.51\n",
      "TP: 8\tFP: 7\tTN: 43\tFN: 42\n",
      "<2019-06-18 02:32:49.602534>  Epoch 9 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5334 - acc: 0.7263\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.824100810587852\n",
      "Acc: 0.7155668254066423\n",
      "TP: 3945\tFP: 2568\tTN: 7625\tFN: 2031\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 02:32:56.519495>  Epoch 10 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5270 - acc: 0.7291\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.482257747232604\n",
      "Acc: 0.7254623044096729\n",
      "TP: 2784\tFP: 1247\tTN: 8946\tFN: 3192\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 02:33:03.468226>  Epoch 11 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5191 - acc: 0.7368\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.281494580954806\n",
      "Acc: 0.7312758983239532\n",
      "TP: 3510\tFP: 1879\tTN: 8314\tFN: 2466\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197085601683607\n",
      "Acc: 0.56\n",
      "TP: 9\tFP: 3\tTN: 47\tFN: 41\n",
      "<2019-06-18 02:33:10.408836>  Epoch 12 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5098 - acc: 0.7409\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.58690700244932\n",
      "Acc: 0.7224318139649948\n",
      "TP: 2327\tFP: 839\tTN: 9354\tFN: 3649\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197077605709305\n",
      "Acc: 0.56\n",
      "TP: 8\tFP: 2\tTN: 48\tFN: 42\n",
      "<2019-06-18 02:33:17.337385>  Epoch 13 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.5017 - acc: 0.7454\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.157593129180428\n",
      "Acc: 0.7348630094625518\n",
      "TP: 3426\tFP: 1737\tTN: 8456\tFN: 2550\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54245737368411\n",
      "Acc: 0.55\n",
      "TP: 6\tFP: 1\tTN: 49\tFN: 44\n",
      "<2019-06-18 02:33:24.275097>  Epoch 14 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4968 - acc: 0.7512\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.435246049527665\n",
      "Acc: 0.7268229327725895\n",
      "TP: 2457\tFP: 898\tTN: 9295\tFN: 3519\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542481361607015\n",
      "Acc: 0.55\n",
      "TP: 9\tFP: 4\tTN: 46\tFN: 41\n",
      "<2019-06-18 02:33:31.185797>  Epoch 15 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4913 - acc: 0.7529\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.875622688638014\n",
      "Acc: 0.7430267796400519\n",
      "TP: 3481\tFP: 1660\tTN: 8533\tFN: 2495\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887885117504725\n",
      "Acc: 0.54\n",
      "TP: 10\tFP: 6\tTN: 44\tFN: 40\n",
      "<2019-06-18 02:33:38.200080>  Epoch 16 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4814 - acc: 0.7610\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.852125815413816\n",
      "Acc: 0.7437070938215103\n",
      "TP: 3499\tFP: 1667\tTN: 8526\tFN: 2477\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54245737368411\n",
      "Acc: 0.55\n",
      "TP: 6\tFP: 1\tTN: 49\tFN: 44\n",
      "<2019-06-18 02:33:45.158359>  Epoch 17 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4761 - acc: 0.7628\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.850006765671626\n",
      "Acc: 0.7437689405652792\n",
      "TP: 3845\tFP: 2012\tTN: 8181\tFN: 2131\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887853133607518\n",
      "Acc: 0.54\n",
      "TP: 6\tFP: 2\tTN: 48\tFN: 44\n",
      "<2019-06-18 02:33:52.140690>  Epoch 18 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4693 - acc: 0.7655\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.73678215922889\n",
      "Acc: 0.7470468179850331\n",
      "TP: 3681\tFP: 1795\tTN: 8398\tFN: 2295\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54245737368411\n",
      "Acc: 0.55\n",
      "TP: 6\tFP: 1\tTN: 49\tFN: 44\n",
      "<2019-06-18 02:33:59.123402>  Epoch 19 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4615 - acc: 0.7708\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.591510648014598\n",
      "Acc: 0.751252396561321\n",
      "TP: 3426\tFP: 1472\tTN: 8721\tFN: 2550\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785896\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:34:06.086600>  Epoch 20 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4548 - acc: 0.7771\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.657761486817886\n",
      "Acc: 0.7493351475044839\n",
      "TP: 4030\tFP: 2107\tTN: 8086\tFN: 1946\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124775009937569\n",
      "Acc: 0.62\n",
      "TP: 17\tFP: 5\tTN: 45\tFN: 33\n",
      "<2019-06-18 02:34:13.012239>  Epoch 21 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4487 - acc: 0.7812\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.702619072959989\n",
      "Acc: 0.7480363658853362\n",
      "TP: 3994\tFP: 2092\tTN: 8101\tFN: 1982\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785896\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:34:19.956087>  Epoch 22 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4422 - acc: 0.7866\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.446251104304672\n",
      "Acc: 0.755457975137609\n",
      "TP: 3413\tFP: 1391\tTN: 8802\tFN: 2563\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785896\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:34:26.882866>  Epoch 23 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4384 - acc: 0.7856\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.538094672853077\n",
      "Acc: 0.7527985651555446\n",
      "TP: 3184\tFP: 1205\tTN: 8988\tFN: 2792\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779355262091256\n",
      "Acc: 0.63\n",
      "TP: 14\tFP: 1\tTN: 49\tFN: 36\n",
      "<2019-06-18 02:34:33.807128>  Epoch 24 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4293 - acc: 0.7941\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.672721136691544\n",
      "Acc: 0.7489022202981013\n",
      "TP: 4162\tFP: 2246\tTN: 7947\tFN: 1814\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.743191970243936\n",
      "Acc: 0.66\n",
      "TP: 17\tFP: 1\tTN: 49\tFN: 33\n",
      "<2019-06-18 02:34:40.743854>  Epoch 25 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4214 - acc: 0.7979\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.48684813952858\n",
      "Acc: 0.7542828870059991\n",
      "TP: 3615\tFP: 1612\tTN: 8581\tFN: 2361\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124743026040361\n",
      "Acc: 0.62\n",
      "TP: 13\tFP: 1\tTN: 49\tFN: 37\n",
      "<2019-06-18 02:34:47.720648>  Epoch 26 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4196 - acc: 0.7993\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.604370139000062\n",
      "Acc: 0.7508813160987075\n",
      "TP: 4286\tFP: 2338\tTN: 7855\tFN: 1690\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779355262091256\n",
      "Acc: 0.63\n",
      "TP: 14\tFP: 1\tTN: 49\tFN: 36\n",
      "<2019-06-18 02:34:54.641493>  Epoch 27 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4133 - acc: 0.8018\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.28390643215424\n",
      "Acc: 0.7601583276640484\n",
      "TP: 3484\tFP: 1386\tTN: 8807\tFN: 2492\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815518553938576\n",
      "Acc: 0.6\n",
      "TP: 11\tFP: 1\tTN: 49\tFN: 39\n",
      "<2019-06-18 02:35:01.610619>  Epoch 28 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.4027 - acc: 0.8093\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.454822746593386\n",
      "Acc: 0.7552105881625333\n",
      "TP: 3959\tFP: 1941\tTN: 8252\tFN: 2017\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785897\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:35:08.533165>  Epoch 29 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3980 - acc: 0.8107\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.388605782750805\n",
      "Acc: 0.7571278372193704\n",
      "TP: 4040\tFP: 1991\tTN: 8202\tFN: 1936\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.052424438320022\n",
      "Acc: 0.68\n",
      "TP: 20\tFP: 2\tTN: 48\tFN: 30\n",
      "<2019-06-18 02:35:15.517357>  Epoch 30 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3958 - acc: 0.8129\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.294590893170476\n",
      "Acc: 0.7598490939452038\n",
      "TP: 3558\tFP: 1465\tTN: 8728\tFN: 2418\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088651697961758\n",
      "Acc: 0.65\n",
      "TP: 25\tFP: 10\tTN: 40\tFN: 25\n",
      "<2019-06-18 02:35:22.469096>  Epoch 31 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3840 - acc: 0.8193\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.249726927656209\n",
      "Acc: 0.7611478755643515\n",
      "TP: 3465\tFP: 1351\tTN: 8842\tFN: 2511\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.052424438320022\n",
      "Acc: 0.68\n",
      "TP: 20\tFP: 2\tTN: 48\tFN: 30\n",
      "<2019-06-18 02:35:29.455229>  Epoch 32 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3796 - acc: 0.8219\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.313817275524501\n",
      "Acc: 0.7592924732512834\n",
      "TP: 3577\tFP: 1493\tTN: 8700\tFN: 2399\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.052424438320022\n",
      "Acc: 0.68\n",
      "TP: 20\tFP: 2\tTN: 48\tFN: 30\n",
      "<2019-06-18 02:35:36.385179>  Epoch 33 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3725 - acc: 0.8267\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.23904498871734\n",
      "Acc: 0.7614571092831962\n",
      "TP: 3442\tFP: 1323\tTN: 8870\tFN: 2534\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088587730167344\n",
      "Acc: 0.65\n",
      "TP: 17\tFP: 2\tTN: 48\tFN: 33\n",
      "<2019-06-18 02:35:43.366843>  Epoch 34 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3691 - acc: 0.8278\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.395005114957648\n",
      "Acc: 0.7569422969880636\n",
      "TP: 3855\tFP: 1809\tTN: 8384\tFN: 2121\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815518553938576\n",
      "Acc: 0.6\n",
      "TP: 11\tFP: 1\tTN: 49\tFN: 39\n",
      "<2019-06-18 02:35:50.342258>  Epoch 35 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3610 - acc: 0.8326\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.296745845425766\n",
      "Acc: 0.7597872472014349\n",
      "TP: 3938\tFP: 1846\tTN: 8347\tFN: 2038\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.434007478013656\n",
      "Acc: 0.64\n",
      "TP: 20\tFP: 6\tTN: 44\tFN: 30\n",
      "<2019-06-18 02:35:57.307955>  Epoch 36 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3567 - acc: 0.8341\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.286065093346833\n",
      "Acc: 0.7600964809202796\n",
      "TP: 3939\tFP: 1842\tTN: 8351\tFN: 2037\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433991486065054\n",
      "Acc: 0.64\n",
      "TP: 18\tFP: 4\tTN: 46\tFN: 32\n",
      "<2019-06-18 02:36:04.268312>  Epoch 37 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3477 - acc: 0.8398\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.47406149207176\n",
      "Acc: 0.7546539674686128\n",
      "TP: 4228\tFP: 2219\tTN: 7974\tFN: 1748\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124743026040361\n",
      "Acc: 0.62\n",
      "TP: 13\tFP: 1\tTN: 49\tFN: 37\n",
      "<2019-06-18 02:36:11.219757>  Epoch 38 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3429 - acc: 0.8433\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.339481365223339\n",
      "Acc: 0.7585503123260561\n",
      "TP: 4187\tFP: 2115\tTN: 8078\tFN: 1789\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124751022014664\n",
      "Acc: 0.62\n",
      "TP: 14\tFP: 2\tTN: 48\tFN: 36\n",
      "<2019-06-18 02:36:18.177168>  Epoch 39 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3373 - acc: 0.8464\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.215572792289338\n",
      "Acc: 0.7621374234646546\n",
      "TP: 3959\tFP: 1829\tTN: 8364\tFN: 2017\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815526549912876\n",
      "Acc: 0.6\n",
      "TP: 12\tFP: 2\tTN: 48\tFN: 38\n",
      "<2019-06-18 02:36:25.145569>  Epoch 40 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3343 - acc: 0.8479\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.343753240763437\n",
      "Acc: 0.7584266188385181\n",
      "TP: 4178\tFP: 2108\tTN: 8085\tFN: 1798\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 10.70705266631952\n",
      "Acc: 0.69\n",
      "TP: 23\tFP: 4\tTN: 46\tFN: 27\n",
      "<2019-06-18 02:36:32.066216>  Epoch 41 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3258 - acc: 0.8526\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.192071023267902\n",
      "Acc: 0.7628177376461129\n",
      "TP: 3878\tFP: 1737\tTN: 8456\tFN: 2098\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.743215958166838\n",
      "Acc: 0.66\n",
      "TP: 20\tFP: 4\tTN: 46\tFN: 30\n",
      "<2019-06-18 02:36:39.098250>  Epoch 42 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3187 - acc: 0.8563\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.19634438238292\n",
      "Acc: 0.762694044158575\n",
      "TP: 3899\tFP: 1760\tTN: 8433\tFN: 2077\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088595726141646\n",
      "Acc: 0.65\n",
      "TP: 18\tFP: 3\tTN: 47\tFN: 32\n",
      "<2019-06-18 02:36:46.022291>  Epoch 43 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3137 - acc: 0.8583\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.341613173709858\n",
      "Acc: 0.7584884655822871\n",
      "TP: 4099\tFP: 2028\tTN: 8165\tFN: 1877\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088587730167344\n",
      "Acc: 0.65\n",
      "TP: 17\tFP: 2\tTN: 48\tFN: 33\n",
      "<2019-06-18 02:36:52.944992>  Epoch 44 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3066 - acc: 0.8615\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.160030448416014\n",
      "Acc: 0.763745438802647\n",
      "TP: 3915\tFP: 1759\tTN: 8434\tFN: 2061\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088611718090249\n",
      "Acc: 0.65\n",
      "TP: 20\tFP: 5\tTN: 45\tFN: 30\n",
      "<2019-06-18 02:36:59.888206>  Epoch 45 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.3020 - acc: 0.8641\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.405709406425336\n",
      "Acc: 0.7566330632692189\n",
      "TP: 4330\tFP: 2289\tTN: 7904\tFN: 1646\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088619714064551\n",
      "Acc: 0.65\n",
      "TP: 21\tFP: 6\tTN: 44\tFN: 29\n",
      "<2019-06-18 02:37:06.814895>  Epoch 46 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2975 - acc: 0.8674\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.117307786267762\n",
      "Acc: 0.7649823736780259\n",
      "TP: 3926\tFP: 1750\tTN: 8443\tFN: 2050\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433967498142149\n",
      "Acc: 0.64\n",
      "TP: 15\tFP: 1\tTN: 49\tFN: 35\n",
      "<2019-06-18 02:37:13.768598>  Epoch 47 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2899 - acc: 0.8701\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.200614823800592\n",
      "Acc: 0.7625703506710372\n",
      "TP: 3861\tFP: 1724\tTN: 8469\tFN: 2115\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.743191970243934\n",
      "Acc: 0.66\n",
      "TP: 17\tFP: 1\tTN: 49\tFN: 33\n",
      "<2019-06-18 02:37:20.680187>  Epoch 48 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2837 - acc: 0.8759\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.241200781665086\n",
      "Acc: 0.7613952625394274\n",
      "TP: 3839\tFP: 1721\tTN: 8472\tFN: 2137\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433975494116448\n",
      "Acc: 0.64\n",
      "TP: 16\tFP: 2\tTN: 48\tFN: 34\n",
      "<2019-06-18 02:37:27.598200>  Epoch 49 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2843 - acc: 0.8747\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.174983620012515\n",
      "Acc: 0.7633125115962645\n",
      "TP: 3916\tFP: 1767\tTN: 8426\tFN: 2060\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779355262091252\n",
      "Acc: 0.63\n",
      "TP: 14\tFP: 1\tTN: 49\tFN: 36\n",
      "<2019-06-18 02:37:34.543362>  Epoch 50 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2725 - acc: 0.8807\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.26684415076752\n",
      "Acc: 0.7606531016142\n",
      "TP: 4030\tFP: 1924\tTN: 8269\tFN: 1946\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160914313861984\n",
      "Acc: 0.59\n",
      "TP: 11\tFP: 2\tTN: 48\tFN: 39\n",
      "<2019-06-18 02:37:41.465187>  Epoch 51 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2687 - acc: 0.8814\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.333065515882371\n",
      "Acc: 0.7587358525573629\n",
      "TP: 4038\tFP: 1963\tTN: 8230\tFN: 1938\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124743026040361\n",
      "Acc: 0.62\n",
      "TP: 13\tFP: 1\tTN: 49\tFN: 37\n",
      "<2019-06-18 02:37:48.379845>  Epoch 52 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2651 - acc: 0.8850\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.20916109695815\n",
      "Acc: 0.7623229636959614\n",
      "TP: 3894\tFP: 1761\tTN: 8432\tFN: 2082\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.743207962192537\n",
      "Acc: 0.66\n",
      "TP: 19\tFP: 3\tTN: 47\tFN: 31\n",
      "<2019-06-18 02:37:55.336445>  Epoch 53 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2561 - acc: 0.8886\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.638556616300225\n",
      "Acc: 0.7498917681984043\n",
      "TP: 4446\tFP: 2514\tTN: 7679\tFN: 1530\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088611718090249\n",
      "Acc: 0.65\n",
      "TP: 20\tFP: 5\tTN: 45\tFN: 30\n",
      "<2019-06-18 02:38:02.275523>  Epoch 54 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2539 - acc: 0.8901\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.266846969559872\n",
      "Acc: 0.7606531016142\n",
      "TP: 4087\tFP: 1981\tTN: 8212\tFN: 1889\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.743207962192537\n",
      "Acc: 0.66\n",
      "TP: 19\tFP: 3\tTN: 47\tFN: 31\n",
      "<2019-06-18 02:38:09.217885>  Epoch 55 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2497 - acc: 0.8923\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.241207902824708\n",
      "Acc: 0.7613952625394274\n",
      "TP: 3983\tFP: 1865\tTN: 8328\tFN: 1993\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.39782019824343\n",
      "Acc: 0.67\n",
      "TP: 20\tFP: 3\tTN: 47\tFN: 30\n",
      "<2019-06-18 02:38:16.220196>  Epoch 56 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2431 - acc: 0.8949\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.315967480340044\n",
      "Acc: 0.7592306265075144\n",
      "TP: 3861\tFP: 1778\tTN: 8415\tFN: 2115\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124751022014664\n",
      "Acc: 0.62\n",
      "TP: 14\tFP: 2\tTN: 48\tFN: 36\n",
      "<2019-06-18 02:38:23.150150>  Epoch 57 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2357 - acc: 0.8993\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.388600886953563\n",
      "Acc: 0.7571278372193704\n",
      "TP: 3941\tFP: 1892\tTN: 8301\tFN: 2035\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.39781220226913\n",
      "Acc: 0.67\n",
      "TP: 19\tFP: 2\tTN: 48\tFN: 31\n",
      "<2019-06-18 02:38:30.101341>  Epoch 58 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2369 - acc: 0.8973\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.41852493414063\n",
      "Acc: 0.7562619828066053\n",
      "TP: 4301\tFP: 2266\tTN: 7927\tFN: 1675\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 11.052440430268625\n",
      "Acc: 0.68\n",
      "TP: 22\tFP: 4\tTN: 46\tFN: 28\n",
      "<2019-06-18 02:38:37.036558>  Epoch 59 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2299 - acc: 0.9012\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.281800289513864\n",
      "Acc: 0.7602201744078174\n",
      "TP: 4091\tFP: 1992\tTN: 8201\tFN: 1885\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433975494116451\n",
      "Acc: 0.64\n",
      "TP: 16\tFP: 2\tTN: 48\tFN: 34\n",
      "<2019-06-18 02:38:44.016213>  Epoch 60 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2272 - acc: 0.9027\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.228395094996772\n",
      "Acc: 0.7617663430020409\n",
      "TP: 4067\tFP: 1943\tTN: 8250\tFN: 1909\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779347266116952\n",
      "Acc: 0.63\n",
      "TP: 13\tFP: 0\tTN: 50\tFN: 37\n",
      "<2019-06-18 02:38:50.972842>  Epoch 61 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2215 - acc: 0.9047\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.29674183977348\n",
      "Acc: 0.7597872472014349\n",
      "TP: 3857\tFP: 1765\tTN: 8428\tFN: 2119\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433975494116448\n",
      "Acc: 0.64\n",
      "TP: 16\tFP: 2\tTN: 48\tFN: 34\n",
      "<2019-06-18 02:38:57.895065>  Epoch 62 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2166 - acc: 0.9070\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.44629348509493\n",
      "Acc: 0.755457975137609\n",
      "TP: 4270\tFP: 2248\tTN: 7945\tFN: 1706\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433991486065052\n",
      "Acc: 0.64\n",
      "TP: 18\tFP: 4\tTN: 46\tFN: 32\n",
      "<2019-06-18 02:39:04.813346>  Epoch 63 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2131 - acc: 0.9102\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.153626863294397\n",
      "Acc: 0.7639309790339539\n",
      "TP: 4014\tFP: 1855\tTN: 8338\tFN: 1962\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779363258065555\n",
      "Acc: 0.63\n",
      "TP: 15\tFP: 2\tTN: 48\tFN: 35\n",
      "<2019-06-18 02:39:11.800358>  Epoch 64 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2098 - acc: 0.9103\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.247616977173537\n",
      "Acc: 0.7612097223081205\n",
      "TP: 3995\tFP: 1880\tTN: 8313\tFN: 1981\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.81552654991288\n",
      "Acc: 0.6\n",
      "TP: 12\tFP: 2\tTN: 48\tFN: 38\n",
      "<2019-06-18 02:39:18.715843>  Epoch 65 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2052 - acc: 0.9137\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.422792853480933\n",
      "Acc: 0.7561382893190673\n",
      "TP: 4212\tFP: 2179\tTN: 8014\tFN: 1764\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.50628608586249\n",
      "Acc: 0.58\n",
      "TP: 8\tFP: 0\tTN: 50\tFN: 42\n",
      "<2019-06-18 02:39:25.655223>  Epoch 66 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.2015 - acc: 0.9146\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.224105070390138\n",
      "Acc: 0.7618900364895789\n",
      "TP: 3709\tFP: 1583\tTN: 8610\tFN: 2267\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124743026040361\n",
      "Acc: 0.62\n",
      "TP: 13\tFP: 1\tTN: 49\tFN: 37\n",
      "<2019-06-18 02:39:32.635018>  Epoch 67 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1993 - acc: 0.9161\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.5039652142824\n",
      "Acc: 0.7537881130558476\n",
      "TP: 4177\tFP: 2182\tTN: 8011\tFN: 1799\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433975494116451\n",
      "Acc: 0.64\n",
      "TP: 16\tFP: 2\tTN: 48\tFN: 34\n",
      "<2019-06-18 02:39:39.582175>  Epoch 68 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1935 - acc: 0.9172\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.384336083120594\n",
      "Acc: 0.7572515307069083\n",
      "TP: 4093\tFP: 2042\tTN: 8151\tFN: 1883\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.088603722115947\n",
      "Acc: 0.65\n",
      "TP: 19\tFP: 4\tTN: 46\tFN: 31\n",
      "<2019-06-18 02:39:46.570131>  Epoch 69 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1885 - acc: 0.9211\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.275389533780125\n",
      "Acc: 0.7604057146391242\n",
      "TP: 4045\tFP: 1943\tTN: 8250\tFN: 1931\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "<2019-06-18 02:39:53.498206>  Epoch 70 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1896 - acc: 0.9203\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.499689530900005\n",
      "Acc: 0.7539118065433855\n",
      "TP: 4109\tFP: 2112\tTN: 8081\tFN: 1867\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779379250014161\n",
      "Acc: 0.63\n",
      "TP: 17\tFP: 4\tTN: 46\tFN: 33\n",
      "<2019-06-18 02:40:00.443579>  Epoch 71 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1844 - acc: 0.9225\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.279656068450505\n",
      "Acc: 0.7602820211515864\n",
      "TP: 3928\tFP: 1828\tTN: 8365\tFN: 2048\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779363258065557\n",
      "Acc: 0.63\n",
      "TP: 15\tFP: 2\tTN: 48\tFN: 35\n",
      "<2019-06-18 02:40:07.387723>  Epoch 72 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1765 - acc: 0.9267\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.335189609779295\n",
      "Acc: 0.7586740058135939\n",
      "TP: 3794\tFP: 1720\tTN: 8473\tFN: 2182\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124735030066057\n",
      "Acc: 0.62\n",
      "TP: 12\tFP: 0\tTN: 50\tFN: 38\n",
      "<2019-06-18 02:40:14.331956>  Epoch 73 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1756 - acc: 0.9282\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.294609932381968\n",
      "Acc: 0.7598490939452038\n",
      "TP: 3943\tFP: 1850\tTN: 8343\tFN: 2033\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779363258065555\n",
      "Acc: 0.63\n",
      "TP: 15\tFP: 2\tTN: 48\tFN: 35\n",
      "<2019-06-18 02:40:21.279194>  Epoch 74 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1722 - acc: 0.9285\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.343752498975975\n",
      "Acc: 0.7584266188385181\n",
      "TP: 4163\tFP: 2093\tTN: 8100\tFN: 1813\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.43398349009075\n",
      "Acc: 0.64\n",
      "TP: 17\tFP: 3\tTN: 47\tFN: 33\n",
      "<2019-06-18 02:40:28.276320>  Epoch 75 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1693 - acc: 0.9299\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.610779608968869\n",
      "Acc: 0.7506957758674005\n",
      "TP: 4306\tFP: 2361\tTN: 7832\tFN: 1670\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779379250014161\n",
      "Acc: 0.63\n",
      "TP: 17\tFP: 4\tTN: 46\tFN: 33\n",
      "<2019-06-18 02:40:35.212367>  Epoch 76 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1650 - acc: 0.9325\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.28605930740464\n",
      "Acc: 0.7600964809202796\n",
      "TP: 3822\tFP: 1725\tTN: 8468\tFN: 2154\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.779371254039857\n",
      "Acc: 0.63\n",
      "TP: 16\tFP: 3\tTN: 47\tFN: 34\n",
      "<2019-06-18 02:40:42.156202>  Epoch 77 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1625 - acc: 0.9333\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.268980904503776\n",
      "Acc: 0.7605912548704311\n",
      "TP: 4042\tFP: 1937\tTN: 8256\tFN: 1934\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.43398349009075\n",
      "Acc: 0.64\n",
      "TP: 17\tFP: 3\tTN: 47\tFN: 33\n",
      "<2019-06-18 02:40:49.130223>  Epoch 78 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1591 - acc: 0.9354\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.414243118648557\n",
      "Acc: 0.7563856762941431\n",
      "TP: 4109\tFP: 2072\tTN: 8121\tFN: 1867\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160930305810588\n",
      "Acc: 0.59\n",
      "TP: 13\tFP: 4\tTN: 46\tFN: 37\n",
      "<2019-06-18 02:40:56.022886>  Epoch 79 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1581 - acc: 0.9354\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.388605189320836\n",
      "Acc: 0.7571278372193704\n",
      "TP: 4028\tFP: 1979\tTN: 8214\tFN: 1948\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 12.433991486065052\n",
      "Acc: 0.64\n",
      "TP: 18\tFP: 4\tTN: 46\tFN: 32\n",
      "<2019-06-18 02:41:02.989685>  Epoch 80 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1574 - acc: 0.9350\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.279664228112573\n",
      "Acc: 0.7602820211515864\n",
      "TP: 4093\tFP: 1993\tTN: 8200\tFN: 1883\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124751022014664\n",
      "Acc: 0.62\n",
      "TP: 14\tFP: 2\tTN: 48\tFN: 36\n",
      "<2019-06-18 02:41:09.945336>  Epoch 81 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1488 - acc: 0.9401\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.399281243412519\n",
      "Acc: 0.7568186035005257\n",
      "TP: 3932\tFP: 1888\tTN: 8305\tFN: 2044\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506302077811092\n",
      "Acc: 0.58\n",
      "TP: 10\tFP: 2\tTN: 48\tFN: 40\n",
      "<2019-06-18 02:41:16.864941>  Epoch 82 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1480 - acc: 0.9399\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.24761668045855\n",
      "Acc: 0.7612097223081205\n",
      "TP: 3989\tFP: 1874\tTN: 8319\tFN: 1987\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "<2019-06-18 02:41:23.821264>  Epoch 83 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1474 - acc: 0.9392\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.26041396020231\n",
      "Acc: 0.7608386418455069\n",
      "TP: 3591\tFP: 1482\tTN: 8711\tFN: 2385\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815518553938576\n",
      "Acc: 0.6\n",
      "TP: 11\tFP: 1\tTN: 49\tFN: 39\n",
      "<2019-06-18 02:41:30.766959>  Epoch 84 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1477 - acc: 0.9400\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.262561939655471\n",
      "Acc: 0.7607767951017379\n",
      "TP: 3830\tFP: 1722\tTN: 8471\tFN: 2146\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "<2019-06-18 02:41:37.692380>  Epoch 85 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1427 - acc: 0.9430\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.36297413389025\n",
      "Acc: 0.7578699981445977\n",
      "TP: 4086\tFP: 2025\tTN: 8168\tFN: 1890\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "<2019-06-18 02:41:44.611799>  Epoch 86 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1409 - acc: 0.9429\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.279659085052845\n",
      "Acc: 0.7602820211515864\n",
      "TP: 3989\tFP: 1889\tTN: 8304\tFN: 1987\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.124759017988962\n",
      "Acc: 0.62\n",
      "TP: 15\tFP: 3\tTN: 47\tFN: 35\n",
      "<2019-06-18 02:41:51.593432>  Epoch 87 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1376 - acc: 0.9438\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.249749181280032\n",
      "Acc: 0.7611478755643515\n",
      "TP: 3915\tFP: 1801\tTN: 8392\tFN: 2061\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160930305810586\n",
      "Acc: 0.59\n",
      "TP: 13\tFP: 4\tTN: 46\tFN: 37\n",
      "<2019-06-18 02:41:58.588477>  Epoch 88 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1324 - acc: 0.9467\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.281794602476666\n",
      "Acc: 0.7602201744078174\n",
      "TP: 3976\tFP: 1877\tTN: 8316\tFN: 2000\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851705833708802\n",
      "Acc: 0.57\n",
      "TP: 11\tFP: 4\tTN: 46\tFN: 39\n",
      "<2019-06-18 02:42:05.534536>  Epoch 89 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1351 - acc: 0.9458\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.456964593936874\n",
      "Acc: 0.7551487414187643\n",
      "TP: 4074\tFP: 2057\tTN: 8136\tFN: 1902\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160922309836288\n",
      "Acc: 0.59\n",
      "TP: 12\tFP: 3\tTN: 47\tFN: 38\n",
      "<2019-06-18 02:42:12.453240>  Epoch 90 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1315 - acc: 0.9475\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.147202013598903\n",
      "Acc: 0.7641165192652607\n",
      "TP: 3683\tFP: 1521\tTN: 8672\tFN: 2293\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785897\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:42:19.449165>  Epoch 91 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1299 - acc: 0.9482\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.388596584586292\n",
      "Acc: 0.7571278372193704\n",
      "TP: 3854\tFP: 1805\tTN: 8388\tFN: 2122\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506302077811092\n",
      "Acc: 0.58\n",
      "TP: 10\tFP: 2\tTN: 48\tFN: 40\n",
      "<2019-06-18 02:42:26.398258>  Epoch 92 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1267 - acc: 0.9500\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.439884658008591\n",
      "Acc: 0.7556435153689158\n",
      "TP: 4263\tFP: 2238\tTN: 7955\tFN: 1713\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160906317887685\n",
      "Acc: 0.59\n",
      "TP: 10\tFP: 1\tTN: 49\tFN: 40\n",
      "<2019-06-18 02:42:33.317225>  Epoch 93 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1274 - acc: 0.9485\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.324511676492714\n",
      "Acc: 0.7589832395324386\n",
      "TP: 3852\tFP: 1773\tTN: 8420\tFN: 2124\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785897\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:42:40.286341>  Epoch 94 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1223 - acc: 0.9510\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.217712216460452\n",
      "Acc: 0.7620755767208857\n",
      "TP: 4025\tFP: 1896\tTN: 8297\tFN: 1951\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "<2019-06-18 02:42:47.238419>  Epoch 95 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1211 - acc: 0.9523\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.215572940646831\n",
      "Acc: 0.7621374234646546\n",
      "TP: 3962\tFP: 1832\tTN: 8361\tFN: 2014\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815558533810085\n",
      "Acc: 0.6\n",
      "TP: 16\tFP: 6\tTN: 44\tFN: 34\n",
      "<2019-06-18 02:42:54.200888>  Epoch 96 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1223 - acc: 0.9514\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.384338407387972\n",
      "Acc: 0.7572515307069083\n",
      "TP: 4140\tFP: 2089\tTN: 8104\tFN: 1836\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815534545887179\n",
      "Acc: 0.6\n",
      "TP: 13\tFP: 3\tTN: 47\tFN: 37\n",
      "<2019-06-18 02:43:01.109773>  Epoch 97 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1173 - acc: 0.9535\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.080988560883636\n",
      "Acc: 0.7660337683220978\n",
      "TP: 3835\tFP: 1642\tTN: 8551\tFN: 2141\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851713829683103\n",
      "Acc: 0.57\n",
      "TP: 12\tFP: 5\tTN: 45\tFN: 38\n",
      "<2019-06-18 02:43:08.074331>  Epoch 98 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1168 - acc: 0.9540\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.435604523901432\n",
      "Acc: 0.7557672088564537\n",
      "TP: 4105\tFP: 2078\tTN: 8115\tFN: 1871\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851713829683103\n",
      "Acc: 0.57\n",
      "TP: 12\tFP: 5\tTN: 45\tFN: 38\n",
      "<2019-06-18 02:43:15.021651>  Epoch 99 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1164 - acc: 0.9541\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.397154874700716\n",
      "Acc: 0.7568804502442946\n",
      "TP: 4130\tFP: 2085\tTN: 8108\tFN: 1846\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.88786112958182\n",
      "Acc: 0.54\n",
      "TP: 7\tFP: 3\tTN: 47\tFN: 43\n",
      "<2019-06-18 02:43:21.987654>  Epoch 100 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1127 - acc: 0.9562\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.279657601477924\n",
      "Acc: 0.7602820211515864\n",
      "TP: 3959\tFP: 1859\tTN: 8334\tFN: 2017\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197085601683607\n",
      "Acc: 0.56\n",
      "TP: 9\tFP: 3\tTN: 47\tFN: 41\n",
      "<2019-06-18 02:43:28.970074>  Epoch 101 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1113 - acc: 0.9565\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.177109246936856\n",
      "Acc: 0.7632506648524955\n",
      "TP: 3703\tFP: 1555\tTN: 8638\tFN: 2273\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.54245737368411\n",
      "Acc: 0.55\n",
      "TP: 6\tFP: 1\tTN: 49\tFN: 44\n",
      "<2019-06-18 02:43:35.929871>  Epoch 102 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1114 - acc: 0.9561\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.320247169374728\n",
      "Acc: 0.7591069330199764\n",
      "TP: 4010\tFP: 1929\tTN: 8264\tFN: 1966\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851689841760198\n",
      "Acc: 0.57\n",
      "TP: 9\tFP: 2\tTN: 48\tFN: 41\n",
      "<2019-06-18 02:43:42.868635>  Epoch 103 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1083 - acc: 0.9578\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.390744366229464\n",
      "Acc: 0.7570659904756014\n",
      "TP: 4089\tFP: 2041\tTN: 8152\tFN: 1887\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197093597657908\n",
      "Acc: 0.56\n",
      "TP: 10\tFP: 4\tTN: 46\tFN: 40\n",
      "<2019-06-18 02:43:49.804381>  Epoch 104 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1044 - acc: 0.9596\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.202747819147046\n",
      "Acc: 0.7625085039272682\n",
      "TP: 3797\tFP: 1661\tTN: 8532\tFN: 2179\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197069609735003\n",
      "Acc: 0.56\n",
      "TP: 7\tFP: 1\tTN: 49\tFN: 43\n",
      "<2019-06-18 02:43:56.784319>  Epoch 105 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1028 - acc: 0.9594\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.416377597569934\n",
      "Acc: 0.7563238295503741\n",
      "TP: 4075\tFP: 2039\tTN: 8154\tFN: 1901\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.542465369658412\n",
      "Acc: 0.55\n",
      "TP: 7\tFP: 2\tTN: 48\tFN: 43\n",
      "<2019-06-18 02:44:03.730943>  Epoch 106 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1068 - acc: 0.9574\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.243342727913566\n",
      "Acc: 0.7613334157956584\n",
      "TP: 3956\tFP: 1839\tTN: 8354\tFN: 2020\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 02:44:10.667857>  Epoch 107 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1017 - acc: 0.9603\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.20702859613667\n",
      "Acc: 0.7623848104397304\n",
      "TP: 3968\tFP: 1834\tTN: 8359\tFN: 2008\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 02:44:17.640208>  Epoch 108 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.1023 - acc: 0.9610\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.311697187279865\n",
      "Acc: 0.7593543199950522\n",
      "TP: 3902\tFP: 1817\tTN: 8376\tFN: 2074\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.887853133607518\n",
      "Acc: 0.54\n",
      "TP: 6\tFP: 2\tTN: 48\tFN: 44\n",
      "<2019-06-18 02:44:24.572790>  Epoch 109 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0988 - acc: 0.9616\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.506103105426094\n",
      "Acc: 0.7537262663120786\n",
      "TP: 4212\tFP: 2218\tTN: 7975\tFN: 1764\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197069609735003\n",
      "Acc: 0.56\n",
      "TP: 7\tFP: 1\tTN: 49\tFN: 43\n",
      "<2019-06-18 02:44:31.525907>  Epoch 110 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0973 - acc: 0.9623\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.395017082462013\n",
      "Acc: 0.7569422969880636\n",
      "TP: 4097\tFP: 2051\tTN: 8142\tFN: 1879\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815526549912876\n",
      "Acc: 0.6\n",
      "TP: 12\tFP: 2\tTN: 48\tFN: 38\n",
      "<2019-06-18 02:44:38.506434>  Epoch 111 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0961 - acc: 0.9628\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.31597267285227\n",
      "Acc: 0.7592306265075144\n",
      "TP: 3966\tFP: 1883\tTN: 8310\tFN: 2010\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197077605709305\n",
      "Acc: 0.56\n",
      "TP: 8\tFP: 2\tTN: 48\tFN: 42\n",
      "<2019-06-18 02:44:45.505002>  Epoch 112 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0954 - acc: 0.9629\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.52533512536482\n",
      "Acc: 0.7531696456181582\n",
      "TP: 4345\tFP: 2360\tTN: 7833\tFN: 1631\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160906317887685\n",
      "Acc: 0.59\n",
      "TP: 10\tFP: 1\tTN: 49\tFN: 40\n",
      "<2019-06-18 02:44:52.424208>  Epoch 113 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0946 - acc: 0.9630\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.536013750986369\n",
      "Acc: 0.7528604118993135\n",
      "TP: 4301\tFP: 2321\tTN: 7872\tFN: 1675\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851689841760198\n",
      "Acc: 0.57\n",
      "TP: 9\tFP: 2\tTN: 48\tFN: 41\n",
      "<2019-06-18 02:44:59.342740>  Epoch 114 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0929 - acc: 0.9641\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.401425761190863\n",
      "Acc: 0.7567567567567568\n",
      "TP: 4101\tFP: 2058\tTN: 8135\tFN: 1875\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506302077811092\n",
      "Acc: 0.58\n",
      "TP: 10\tFP: 2\tTN: 48\tFN: 40\n",
      "<2019-06-18 02:45:06.303226>  Epoch 115 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0948 - acc: 0.9635\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.439878921518893\n",
      "Acc: 0.7556435153689158\n",
      "TP: 4147\tFP: 2122\tTN: 8071\tFN: 1829\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851681845785897\n",
      "Acc: 0.57\n",
      "TP: 8\tFP: 1\tTN: 49\tFN: 42\n",
      "<2019-06-18 02:45:13.224671>  Epoch 116 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0940 - acc: 0.9634\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.230521710971065\n",
      "Acc: 0.761704496258272\n",
      "TP: 3874\tFP: 1751\tTN: 8442\tFN: 2102\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851689841760198\n",
      "Acc: 0.57\n",
      "TP: 9\tFP: 2\tTN: 48\tFN: 41\n",
      "<2019-06-18 02:45:20.225664>  Epoch 117 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0870 - acc: 0.9669\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.362975073487702\n",
      "Acc: 0.7578699981445977\n",
      "TP: 4105\tFP: 2044\tTN: 8149\tFN: 1871\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815534545887179\n",
      "Acc: 0.6\n",
      "TP: 13\tFP: 3\tTN: 47\tFN: 37\n",
      "<2019-06-18 02:45:27.146584>  Epoch 118 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0907 - acc: 0.9654\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.322383725300991\n",
      "Acc: 0.7590450862762076\n",
      "TP: 4018\tFP: 1938\tTN: 8255\tFN: 1958\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160922309836288\n",
      "Acc: 0.59\n",
      "TP: 12\tFP: 3\tTN: 47\tFN: 38\n",
      "<2019-06-18 02:45:34.101537>  Epoch 119 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0895 - acc: 0.9661\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.29889001703663\n",
      "Acc: 0.7597254004576659\n",
      "TP: 4100\tFP: 2009\tTN: 8184\tFN: 1876\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.851713829683103\n",
      "Acc: 0.57\n",
      "TP: 12\tFP: 5\tTN: 45\tFN: 38\n",
      "<2019-06-18 02:45:41.042265>  Epoch 120 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0871 - acc: 0.9664\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.397154825248215\n",
      "Acc: 0.7568804502442946\n",
      "TP: 4129\tFP: 2084\tTN: 8109\tFN: 1847\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.815518553938578\n",
      "Acc: 0.6\n",
      "TP: 11\tFP: 1\tTN: 49\tFN: 39\n",
      "<2019-06-18 02:45:47.978327>  Epoch 121 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0837 - acc: 0.9684\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.256164931716006\n",
      "Acc: 0.7609623353330447\n",
      "TP: 4062\tFP: 1951\tTN: 8242\tFN: 1914\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 13.81554254186148\n",
      "Acc: 0.6\n",
      "TP: 14\tFP: 4\tTN: 46\tFN: 36\n",
      "<2019-06-18 02:45:54.937272>  Epoch 122 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0836 - acc: 0.9680\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.333064971904896\n",
      "Acc: 0.7587358525573629\n",
      "TP: 4027\tFP: 1952\tTN: 8241\tFN: 1949\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "<2019-06-18 02:46:01.896692>  Epoch 123 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0838 - acc: 0.9681\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.401422101706055\n",
      "Acc: 0.7567567567567568\n",
      "TP: 4027\tFP: 1984\tTN: 8209\tFN: 1949\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160930305810588\n",
      "Acc: 0.59\n",
      "TP: 13\tFP: 4\tTN: 46\tFN: 37\n",
      "<2019-06-18 02:46:08.858012>  Epoch 124 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0830 - acc: 0.9683\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.482604550816987\n",
      "Acc: 0.754406580493537\n",
      "TP: 4196\tFP: 2191\tTN: 8002\tFN: 1780\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.197085601683607\n",
      "Acc: 0.56\n",
      "TP: 9\tFP: 3\tTN: 47\tFN: 41\n",
      "<2019-06-18 02:46:15.789892>  Epoch 125 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0801 - acc: 0.9693\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.422791172096023\n",
      "Acc: 0.7561382893190673\n",
      "TP: 4178\tFP: 2145\tTN: 8048\tFN: 1798\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.160930305810586\n",
      "Acc: 0.59\n",
      "TP: 13\tFP: 4\tTN: 46\tFN: 37\n",
      "<2019-06-18 02:46:22.746453>  Epoch 126 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0821 - acc: 0.9690\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.352283936384318\n",
      "Acc: 0.7581792318634424\n",
      "TP: 3896\tFP: 1830\tTN: 8363\tFN: 2080\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506326065733994\n",
      "Acc: 0.58\n",
      "TP: 13\tFP: 5\tTN: 45\tFN: 37\n",
      "<2019-06-18 02:46:29.693220>  Epoch 127 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0804 - acc: 0.9694\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.3095700767806\n",
      "Acc: 0.7594161667388212\n",
      "TP: 4085\tFP: 1999\tTN: 8194\tFN: 1891\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 15.19710159363221\n",
      "Acc: 0.56\n",
      "TP: 11\tFP: 5\tTN: 45\tFN: 39\n",
      "<2019-06-18 02:46:36.646447>  Epoch 128 of 128\n",
      "Epoch 1/1\n",
      "64679/64679 [==============================] - 6s 92us/step - loss: 0.0815 - acc: 0.9679\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.414235107343984\n",
      "Acc: 0.7563856762941431\n",
      "TP: 3947\tFP: 1910\tTN: 8283\tFN: 2029\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n"
     ]
    }
   ],
   "source": [
    "model = build_model_concat_LSTM()\n",
    "\n",
    "bestEN = sys.float_info.max\n",
    "bestES = sys.float_info.max\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    printTrace('Epoch ' +str(epoch+1) + ' of ' +str(EPOCHS))\n",
    "    model.fit(train_set.X, train_set.y, batch_size=BATCH_SIZE, epochs=1, verbose=1)\n",
    "    print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "    \n",
    "    result_en = evaluate(model,dev_set)\n",
    "    \n",
    "    if result_en < bestEN:\n",
    "        bestEN = result_en\n",
    "        model.save(Save_path+'bestEN_concat_LSTM.h5')\n",
    "    \n",
    "    print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "    result_es = evaluate(model,test_set)\n",
    "    if result_es < bestES:\n",
    "        bestEN = result_en\n",
    "        model.save(Save_path+'bestES_concat_LSTM.h5')\n",
    "\n",
    "    \n",
    "model.save(Save_path+'final_concat_LSTM.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ===== LAST: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.414235107343984\n",
      "Acc: 0.7563856762941431\n",
      "TP: 3947\tFP: 1910\tTN: 8283\tFN: 2029\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "\n",
      "\n",
      "\n",
      " ===== BEST IN DEV SET: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.414235107343984\n",
      "Acc: 0.7563856762941431\n",
      "TP: 3947\tFP: 1910\tTN: 8283\tFN: 2029\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n",
      "\n",
      "\n",
      "\n",
      " ===== BEST IN TEST SET: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.414235107343984\n",
      "Acc: 0.7563856762941431\n",
      "TP: 3947\tFP: 1910\tTN: 8283\tFN: 2029\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 14.506318069759695\n",
      "Acc: 0.58\n",
      "TP: 12\tFP: 4\tTN: 46\tFN: 38\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "14.506318069759695"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\" ===== LAST: ====\")\n",
    "mode = load_model(Save_path+'final_concat_LSTM.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\" ===== BEST IN DEV SET: ====\")\n",
    "mode = load_model(Save_path+'bestEN_concat_LSTM.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\" ===== BEST IN TEST SET: ====\")\n",
    "mode = load_model(Save_path+'bestES_concat_LSTM.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RUN LSTM CONCAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 25, 300)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            (None, 25, 300)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 256)          439296      input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional) (None, 256)          439296      input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 512)          0           bidirectional_3[0][0]            \n",
      "                                                                 bidirectional_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 256)          131328      concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 1)            257         dense_3[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 1,010,177\n",
      "Trainable params: 1,010,177\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "<2019-06-18 02:46:56.390222>  Epoch 1 of 128\n",
      "Epoch 1/1\n",
      " - 12s - loss: 0.6291 - acc: 0.6450\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 12.218733892952272\n",
      "Acc: 0.64623662564166\n",
      "TP: 3892\tFP: 3636\tTN: 6557\tFN: 2084\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:47:14.115613>  Epoch 2 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.6005 - acc: 0.6737\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.693464201476667\n",
      "Acc: 0.6903952006926836\n",
      "TP: 2856\tFP: 1886\tTN: 8307\tFN: 3120\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:47:24.053173>  Epoch 3 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5880 - acc: 0.6866\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.520434821047509\n",
      "Acc: 0.6954047869379677\n",
      "TP: 2848\tFP: 1797\tTN: 8396\tFN: 3128\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:47:33.957005>  Epoch 4 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5826 - acc: 0.6909\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.326041661645604\n",
      "Acc: 0.7010328406209413\n",
      "TP: 2796\tFP: 1654\tTN: 8539\tFN: 3180\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:47:43.844137>  Epoch 5 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5772 - acc: 0.6952\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.471259516900234\n",
      "Acc: 0.6968272620446534\n",
      "TP: 1966\tFP: 892\tTN: 9301\tFN: 4010\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:47:53.684946>  Epoch 6 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5714 - acc: 0.6998\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.311076571997235\n",
      "Acc: 0.7014657678273238\n",
      "TP: 2554\tFP: 1405\tTN: 8788\tFN: 3422\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:48:03.612131>  Epoch 7 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5677 - acc: 0.7044\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.176487593151784\n",
      "Acc: 0.7053621126847671\n",
      "TP: 2334\tFP: 1122\tTN: 9071\tFN: 3642\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:48:13.535600>  Epoch 8 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5641 - acc: 0.7051\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.118820611404061\n",
      "Acc: 0.7070319747665286\n",
      "TP: 2523\tFP: 1284\tTN: 8909\tFN: 3453\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:48:23.439808>  Epoch 9 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5612 - acc: 0.7086\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.165837007096252\n",
      "Acc: 0.7056713464036118\n",
      "TP: 2945\tFP: 1728\tTN: 8465\tFN: 3031\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:48:33.284755>  Epoch 10 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5575 - acc: 0.7097\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.182935141543574\n",
      "Acc: 0.7051765724534603\n",
      "TP: 3124\tFP: 1915\tTN: 8278\tFN: 2852\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:48:43.131661>  Epoch 11 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5550 - acc: 0.7117\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.035499677719468\n",
      "Acc: 0.7094439977735172\n",
      "TP: 2307\tFP: 1029\tTN: 9164\tFN: 3669\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:48:53.023286>  Epoch 12 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5493 - acc: 0.7153\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.984261353509536\n",
      "Acc: 0.7109283196239718\n",
      "TP: 2904\tFP: 1602\tTN: 8591\tFN: 3072\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:49:02.910663>  Epoch 13 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5464 - acc: 0.7183\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.943671192182766\n",
      "Acc: 0.7121034077555817\n",
      "TP: 2841\tFP: 1520\tTN: 8673\tFN: 3135\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:49:12.808230>  Epoch 14 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5434 - acc: 0.7203\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.12306424956815\n",
      "Acc: 0.7069082812789906\n",
      "TP: 1943\tFP: 706\tTN: 9487\tFN: 4033\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:49:22.649568>  Epoch 15 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5410 - acc: 0.7224\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 10.076154176745336\n",
      "Acc: 0.7082689096419074\n",
      "TP: 3671\tFP: 2412\tTN: 7781\tFN: 2305\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:49:32.550588>  Epoch 16 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5350 - acc: 0.7286\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.755700706566467\n",
      "Acc: 0.7175459212072485\n",
      "TP: 3076\tFP: 1667\tTN: 8526\tFN: 2900\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:49:42.440287>  Epoch 17 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5309 - acc: 0.7290\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.777027396166176\n",
      "Acc: 0.716927453769559\n",
      "TP: 2370\tFP: 971\tTN: 9222\tFN: 3606\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:49:52.273083>  Epoch 18 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5294 - acc: 0.7319\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.75145607935243\n",
      "Acc: 0.7176696146947863\n",
      "TP: 3636\tFP: 2225\tTN: 7968\tFN: 2340\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:50:02.176159>  Epoch 19 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5253 - acc: 0.7341\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.61897744660963\n",
      "Acc: 0.7215041128084606\n",
      "TP: 2894\tFP: 1421\tTN: 8772\tFN: 3082\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:50:12.068141>  Epoch 20 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5199 - acc: 0.7375\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.492943642371266\n",
      "Acc: 0.7251530706908281\n",
      "TP: 2887\tFP: 1355\tTN: 8838\tFN: 3089\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:50:21.952222>  Epoch 21 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5197 - acc: 0.7377\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.49507208808796\n",
      "Acc: 0.7250912239470592\n",
      "TP: 2731\tFP: 1200\tTN: 8993\tFN: 3245\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:50:31.786281>  Epoch 22 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 9s - loss: 0.5145 - acc: 0.7422\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.38828054045528\n",
      "Acc: 0.7281835611355062\n",
      "TP: 3064\tFP: 1483\tTN: 8710\tFN: 2912\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:50:41.672258>  Epoch 23 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5112 - acc: 0.7446\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.435281111348308\n",
      "Acc: 0.7268229327725895\n",
      "TP: 3166\tFP: 1607\tTN: 8586\tFN: 2810\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:50:51.505778>  Epoch 24 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5067 - acc: 0.7460\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.501516372614924\n",
      "Acc: 0.7249056837157524\n",
      "TP: 3455\tFP: 1927\tTN: 8266\tFN: 2521\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:51:01.379678>  Epoch 25 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.5030 - acc: 0.7473\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.324205226146193\n",
      "Acc: 0.7300389634485744\n",
      "TP: 3256\tFP: 1645\tTN: 8548\tFN: 2720\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:51:11.259262>  Epoch 26 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4985 - acc: 0.7511\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.341280711349713\n",
      "Acc: 0.7295441894984229\n",
      "TP: 2977\tFP: 1374\tTN: 8819\tFN: 2999\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:51:21.091813>  Epoch 27 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4961 - acc: 0.7522\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.174665251614126\n",
      "Acc: 0.7343682355124003\n",
      "TP: 3079\tFP: 1398\tTN: 8795\tFN: 2897\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:51:30.968788>  Epoch 28 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4911 - acc: 0.7568\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.275071412644225\n",
      "Acc: 0.73146143855526\n",
      "TP: 3213\tFP: 1579\tTN: 8614\tFN: 2763\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:51:40.815578>  Epoch 29 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4916 - acc: 0.7567\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.418211313181988\n",
      "Acc: 0.7273177067227411\n",
      "TP: 3560\tFP: 1993\tTN: 8200\tFN: 2416\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:51:50.776688>  Epoch 30 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4837 - acc: 0.7599\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.249438527471234\n",
      "Acc: 0.7322035994804873\n",
      "TP: 3234\tFP: 1588\tTN: 8605\tFN: 2742\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:52:00.687282>  Epoch 31 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4805 - acc: 0.7626\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.396858611568655\n",
      "Acc: 0.7279361741604304\n",
      "TP: 3740\tFP: 2163\tTN: 8030\tFN: 2236\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:52:10.533219>  Epoch 32 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4768 - acc: 0.7644\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.349863777165325\n",
      "Acc: 0.7292968025233472\n",
      "TP: 3754\tFP: 2155\tTN: 8038\tFN: 2222\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:52:20.450635>  Epoch 33 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4727 - acc: 0.7668\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.198191301811775\n",
      "Acc: 0.733687921330942\n",
      "TP: 3651\tFP: 1981\tTN: 8212\tFN: 2325\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:52:30.343663>  Epoch 34 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4696 - acc: 0.7679\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.131948968838035\n",
      "Acc: 0.7356051703877791\n",
      "TP: 3219\tFP: 1518\tTN: 8675\tFN: 2757\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:52:40.235216>  Epoch 35 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4635 - acc: 0.7719\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.1469103990016\n",
      "Acc: 0.7351722431813965\n",
      "TP: 3387\tFP: 1693\tTN: 8500\tFN: 2589\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 02:52:50.059389>  Epoch 36 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4632 - acc: 0.7717\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.166116703641688\n",
      "Acc: 0.734615622487476\n",
      "TP: 3000\tFP: 1315\tTN: 8878\tFN: 2976\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:52:59.886313>  Epoch 37 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4567 - acc: 0.7767\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.157562369727057\n",
      "Acc: 0.7348630094625518\n",
      "TP: 2804\tFP: 1115\tTN: 9078\tFN: 3172\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:53:09.776135>  Epoch 38 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4538 - acc: 0.7784\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.119125677080653\n",
      "Acc: 0.7359762508503928\n",
      "TP: 3091\tFP: 1384\tTN: 8809\tFN: 2885\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:53:19.648289>  Epoch 39 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4497 - acc: 0.7802\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.42676960329641\n",
      "Acc: 0.7270703197476653\n",
      "TP: 3836\tFP: 2273\tTN: 7920\tFN: 2140\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:53:29.510526>  Epoch 40 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4448 - acc: 0.7821\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.275094259698015\n",
      "Acc: 0.73146143855526\n",
      "TP: 3675\tFP: 2041\tTN: 8152\tFN: 2301\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578636657480033\n",
      "Acc: 0.52\n",
      "TP: 5\tFP: 3\tTN: 47\tFN: 45\n",
      "<2019-06-18 02:53:39.412113>  Epoch 41 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4385 - acc: 0.7882\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.031544884812828\n",
      "Acc: 0.7385119673449193\n",
      "TP: 3127\tFP: 1379\tTN: 8814\tFN: 2849\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614783957378748\n",
      "Acc: 0.49\n",
      "TP: 0\tFP: 1\tTN: 49\tFN: 50\n",
      "<2019-06-18 02:53:49.332125>  Epoch 42 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4346 - acc: 0.7880\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.121289530785475\n",
      "Acc: 0.7359144041066238\n",
      "TP: 3651\tFP: 1945\tTN: 8248\tFN: 2325\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:53:59.175456>  Epoch 43 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4296 - acc: 0.7908\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.052920730194934\n",
      "Acc: 0.7378934999072299\n",
      "TP: 3415\tFP: 1677\tTN: 8516\tFN: 2561\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 02:54:09.094069>  Epoch 44 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4270 - acc: 0.7924\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.069985978731497\n",
      "Acc: 0.7373987259570783\n",
      "TP: 2929\tFP: 1199\tTN: 8994\tFN: 3047\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269388197455342\n",
      "Acc: 0.5\n",
      "TP: 0\tFP: 0\tTN: 50\tFN: 50\n",
      "<2019-06-18 02:54:18.936226>  Epoch 45 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4186 - acc: 0.7992\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.108465200525647\n",
      "Acc: 0.7362854845692375\n",
      "TP: 3502\tFP: 1790\tTN: 8403\tFN: 2474\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:54:28.784641>  Epoch 46 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4176 - acc: 0.7984\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.044382369436958\n",
      "Acc: 0.7381408868823056\n",
      "TP: 3542\tFP: 1800\tTN: 8393\tFN: 2434\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:54:38.676253>  Epoch 47 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4144 - acc: 0.7988\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.138370455763708\n",
      "Acc: 0.7354196301564723\n",
      "TP: 3482\tFP: 1784\tTN: 8409\tFN: 2494\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:54:48.519713>  Epoch 48 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4075 - acc: 0.8054\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.095610902052407\n",
      "Acc: 0.7366565650318511\n",
      "TP: 2747\tFP: 1029\tTN: 9164\tFN: 3229\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:54:58.438241>  Epoch 49 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.4075 - acc: 0.8065\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.228097991172259\n",
      "Acc: 0.7328220669181768\n",
      "TP: 3660\tFP: 2004\tTN: 8189\tFN: 2316\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:55:08.294067>  Epoch 50 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 9s - loss: 0.4003 - acc: 0.8080\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.332777362959883\n",
      "Acc: 0.7297915764734987\n",
      "TP: 3812\tFP: 2205\tTN: 7988\tFN: 2164\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614791953353052\n",
      "Acc: 0.49\n",
      "TP: 1\tFP: 2\tTN: 48\tFN: 49\n",
      "<2019-06-18 02:55:18.131358>  Epoch 51 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3931 - acc: 0.8142\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.008058149350598\n",
      "Acc: 0.7391922815263776\n",
      "TP: 3350\tFP: 1591\tTN: 8602\tFN: 2626\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:55:28.023890>  Epoch 52 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3917 - acc: 0.8138\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.982424176222665\n",
      "Acc: 0.7399344424516049\n",
      "TP: 3349\tFP: 1578\tTN: 8615\tFN: 2627\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:55:37.915701>  Epoch 53 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3850 - acc: 0.8180\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.492999573145807\n",
      "Acc: 0.7251530706908281\n",
      "TP: 4018\tFP: 2486\tTN: 7707\tFN: 1958\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:55:47.751507>  Epoch 54 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3814 - acc: 0.8194\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.063612510180786\n",
      "Acc: 0.7375842661883851\n",
      "TP: 3637\tFP: 1904\tTN: 8289\tFN: 2339\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:55:57.644237>  Epoch 55 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3784 - acc: 0.8221\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.082833057140121\n",
      "Acc: 0.7370276454944648\n",
      "TP: 3538\tFP: 1814\tTN: 8379\tFN: 2438\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.57862066553143\n",
      "Acc: 0.52\n",
      "TP: 3\tFP: 1\tTN: 49\tFN: 47\n",
      "<2019-06-18 02:56:07.487199>  Epoch 56 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3727 - acc: 0.8254\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.106322759752194\n",
      "Acc: 0.7363473313130063\n",
      "TP: 3375\tFP: 1662\tTN: 8531\tFN: 2601\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.57862066553143\n",
      "Acc: 0.52\n",
      "TP: 3\tFP: 1\tTN: 49\tFN: 47\n",
      "<2019-06-18 02:56:17.336648>  Epoch 57 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3662 - acc: 0.8275\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.153322539405265\n",
      "Acc: 0.7349867029500897\n",
      "TP: 3461\tFP: 1770\tTN: 8423\tFN: 2515\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:56:27.173156>  Epoch 58 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3664 - acc: 0.8282\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.178963287525338\n",
      "Acc: 0.7342445420248624\n",
      "TP: 3599\tFP: 1920\tTN: 8273\tFN: 2377\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:56:37.047648>  Epoch 59 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3588 - acc: 0.8327\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.999511332215567\n",
      "Acc: 0.7394396685014534\n",
      "TP: 3306\tFP: 1543\tTN: 8650\tFN: 2670\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:56:46.960226>  Epoch 60 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3543 - acc: 0.8344\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.144777453107643\n",
      "Acc: 0.7352340899251655\n",
      "TP: 3452\tFP: 1757\tTN: 8436\tFN: 2524\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:56:56.778729>  Epoch 61 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3524 - acc: 0.8354\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.456683858888994\n",
      "Acc: 0.7262044653349001\n",
      "TP: 3998\tFP: 2449\tTN: 7744\tFN: 1978\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:57:06.646547>  Epoch 62 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3466 - acc: 0.8392\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.262284864092399\n",
      "Acc: 0.7318325190178737\n",
      "TP: 3828\tFP: 2188\tTN: 8005\tFN: 2148\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578612669557128\n",
      "Acc: 0.52\n",
      "TP: 2\tFP: 0\tTN: 50\tFN: 48\n",
      "<2019-06-18 02:57:16.537237>  Epoch 63 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3445 - acc: 0.8396\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.0379682509334\n",
      "Acc: 0.7383264271136125\n",
      "TP: 3428\tFP: 1683\tTN: 8510\tFN: 2548\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:57:26.417462>  Epoch 64 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3378 - acc: 0.8434\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.969618390649357\n",
      "Acc: 0.7403055229142186\n",
      "TP: 3575\tFP: 1798\tTN: 8395\tFN: 2401\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924008429480537\n",
      "Acc: 0.51\n",
      "TP: 2\tFP: 1\tTN: 49\tFN: 48\n",
      "<2019-06-18 02:57:36.308249>  Epoch 65 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3367 - acc: 0.8436\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.91619440418326\n",
      "Acc: 0.7418516915084421\n",
      "TP: 3171\tFP: 1369\tTN: 8824\tFN: 2805\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:57:46.212247>  Epoch 66 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3296 - acc: 0.8483\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.520769310960045\n",
      "Acc: 0.7243490630218319\n",
      "TP: 4011\tFP: 2492\tTN: 7701\tFN: 1965\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:57:56.063890>  Epoch 67 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3266 - acc: 0.8483\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.416088257787509\n",
      "Acc: 0.72737955346651\n",
      "TP: 3825\tFP: 2257\tTN: 7936\tFN: 2151\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:58:05.964248>  Epoch 68 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3254 - acc: 0.8508\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.097797899526\n",
      "Acc: 0.7365947182880821\n",
      "TP: 3775\tFP: 2058\tTN: 8135\tFN: 2201\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924000433506237\n",
      "Acc: 0.51\n",
      "TP: 1\tFP: 0\tTN: 50\tFN: 49\n",
      "<2019-06-18 02:58:15.860176>  Epoch 69 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3194 - acc: 0.8542\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.001661289768624\n",
      "Acc: 0.7393778217576844\n",
      "TP: 3585\tFP: 1823\tTN: 8370\tFN: 2391\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:58:25.756236>  Epoch 70 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3180 - acc: 0.8550\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.334913078193694\n",
      "Acc: 0.7297297297297297\n",
      "TP: 3803\tFP: 2197\tTN: 7996\tFN: 2173\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614791953353052\n",
      "Acc: 0.49\n",
      "TP: 1\tFP: 2\tTN: 48\tFN: 49\n",
      "<2019-06-18 02:58:35.596698>  Epoch 71 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3097 - acc: 0.8584\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.901252804471147\n",
      "Acc: 0.7422846187148247\n",
      "TP: 3404\tFP: 1595\tTN: 8598\tFN: 2572\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 02:58:45.500244>  Epoch 72 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3059 - acc: 0.8615\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.946099214348841\n",
      "Acc: 0.7409858370956769\n",
      "TP: 3142\tFP: 1354\tTN: 8839\tFN: 2834\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.614791953353052\n",
      "Acc: 0.49\n",
      "TP: 1\tFP: 2\tTN: 48\tFN: 49\n",
      "<2019-06-18 02:58:55.341387>  Epoch 73 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.3017 - acc: 0.8631\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.08282227649569\n",
      "Acc: 0.7370276454944648\n",
      "TP: 3320\tFP: 1596\tTN: 8597\tFN: 2656\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269404189403947\n",
      "Acc: 0.5\n",
      "TP: 2\tFP: 2\tTN: 48\tFN: 48\n",
      "<2019-06-18 02:59:05.179263>  Epoch 74 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2998 - acc: 0.8635\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.164012291838722\n",
      "Acc: 0.734677469231245\n",
      "TP: 3642\tFP: 1956\tTN: 8237\tFN: 2334\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 02:59:15.007811>  Epoch 75 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2955 - acc: 0.8671\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.978157147027312\n",
      "Acc: 0.7400581359391428\n",
      "TP: 3456\tFP: 1683\tTN: 8510\tFN: 2520\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269396193429642\n",
      "Acc: 0.5\n",
      "TP: 1\tFP: 1\tTN: 49\tFN: 49\n",
      "<2019-06-18 02:59:24.923779>  Epoch 76 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2931 - acc: 0.8681\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.30286142598239\n",
      "Acc: 0.7306574308862639\n",
      "TP: 3616\tFP: 1995\tTN: 8198\tFN: 2360\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 02:59:34.776649>  Epoch 77 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2898 - acc: 0.8689\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.92048981911211\n",
      "Acc: 0.7417279980209042\n",
      "TP: 3638\tFP: 1838\tTN: 8355\tFN: 2338\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233240897556627\n",
      "Acc: 0.53\n",
      "TP: 5\tFP: 2\tTN: 48\tFN: 45\n",
      "<2019-06-18 02:59:44.675491>  Epoch 78 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 9s - loss: 0.2856 - acc: 0.8725\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.858528807105465\n",
      "Acc: 0.7435215535902034\n",
      "TP: 3388\tFP: 1559\tTN: 8634\tFN: 2588\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92402442142914\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 02:59:54.582506>  Epoch 79 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2814 - acc: 0.8743\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.012337838385278\n",
      "Acc: 0.7390685880388398\n",
      "TP: 3499\tFP: 1742\tTN: 8451\tFN: 2477\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 03:00:04.416896>  Epoch 80 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2773 - acc: 0.8756\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.11488475880392\n",
      "Acc: 0.7360999443379306\n",
      "TP: 3726\tFP: 2017\tTN: 8176\tFN: 2250\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:00:14.269388>  Epoch 81 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2739 - acc: 0.8770\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.023023832428935\n",
      "Acc: 0.7387593543199951\n",
      "TP: 3604\tFP: 1852\tTN: 8341\tFN: 2372\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:00:24.185766>  Epoch 82 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2683 - acc: 0.8809\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.076422251953884\n",
      "Acc: 0.7372131857257715\n",
      "TP: 3491\tFP: 1764\tTN: 8429\tFN: 2485\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:00:34.027033>  Epoch 83 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2693 - acc: 0.8788\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.093516825821393\n",
      "Acc: 0.73671841177562\n",
      "TP: 3598\tFP: 1879\tTN: 8314\tFN: 2378\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:00:43.859839>  Epoch 84 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2611 - acc: 0.8850\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.388317877090802\n",
      "Acc: 0.7281835611355062\n",
      "TP: 3819\tFP: 2238\tTN: 7955\tFN: 2157\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:00:53.693039>  Epoch 85 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2624 - acc: 0.8845\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.326371354665893\n",
      "Acc: 0.7299771167048055\n",
      "TP: 3862\tFP: 2252\tTN: 7941\tFN: 2114\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269404189403947\n",
      "Acc: 0.5\n",
      "TP: 2\tFP: 2\tTN: 48\tFN: 48\n",
      "<2019-06-18 03:01:03.600237>  Epoch 86 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2578 - acc: 0.8861\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.657490197197008\n",
      "Acc: 0.7203908714206197\n",
      "TP: 4145\tFP: 2690\tTN: 7503\tFN: 1831\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:01:13.450660>  Epoch 87 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2540 - acc: 0.8873\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.982435649202058\n",
      "Acc: 0.7399344424516049\n",
      "TP: 3581\tFP: 1810\tTN: 8383\tFN: 2395\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:01:23.356229>  Epoch 88 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2513 - acc: 0.8883\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.136249477374118\n",
      "Acc: 0.7354814769002412\n",
      "TP: 3789\tFP: 2090\tTN: 8103\tFN: 2187\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:01:33.196448>  Epoch 89 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2459 - acc: 0.8925\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.313552167465794\n",
      "Acc: 0.7303481971674192\n",
      "TP: 3817\tFP: 2201\tTN: 7992\tFN: 2159\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:01:43.028267>  Epoch 90 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2462 - acc: 0.8917\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.052927752449564\n",
      "Acc: 0.7378934999072299\n",
      "TP: 3557\tFP: 1819\tTN: 8374\tFN: 2419\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:01:52.927220>  Epoch 91 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2455 - acc: 0.8926\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.978164812164408\n",
      "Acc: 0.7400581359391428\n",
      "TP: 3611\tFP: 1838\tTN: 8355\tFN: 2365\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924032417403442\n",
      "Acc: 0.51\n",
      "TP: 5\tFP: 4\tTN: 46\tFN: 45\n",
      "<2019-06-18 03:02:02.843225>  Epoch 92 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2368 - acc: 0.8960\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.260151868745945\n",
      "Acc: 0.7318943657616427\n",
      "TP: 3892\tFP: 2251\tTN: 7942\tFN: 2084\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578636657480033\n",
      "Acc: 0.52\n",
      "TP: 5\tFP: 3\tTN: 47\tFN: 45\n",
      "<2019-06-18 03:02:12.672769>  Epoch 93 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2367 - acc: 0.8976\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.025157223395366\n",
      "Acc: 0.7386975075762261\n",
      "TP: 3548\tFP: 1797\tTN: 8396\tFN: 2428\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:02:22.595028>  Epoch 94 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2320 - acc: 0.8997\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.138380494620675\n",
      "Acc: 0.7354196301564723\n",
      "TP: 3685\tFP: 1987\tTN: 8206\tFN: 2291\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:02:32.453264>  Epoch 95 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2342 - acc: 0.8971\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.1298383260204\n",
      "Acc: 0.735667017131548\n",
      "TP: 3735\tFP: 2033\tTN: 8160\tFN: 2241\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924032417403442\n",
      "Acc: 0.51\n",
      "TP: 5\tFP: 4\tTN: 46\tFN: 45\n",
      "<2019-06-18 03:02:42.354363>  Epoch 96 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2303 - acc: 0.9007\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.86921653198653\n",
      "Acc: 0.7432123198713587\n",
      "TP: 3528\tFP: 1704\tTN: 8489\tFN: 2448\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:02:52.235574>  Epoch 97 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2258 - acc: 0.9017\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.92900468993383\n",
      "Acc: 0.7414806110458284\n",
      "TP: 3036\tFP: 1240\tTN: 8953\tFN: 2940\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:03:02.065516>  Epoch 98 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2249 - acc: 0.9016\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.488725422790832\n",
      "Acc: 0.7252767641783661\n",
      "TP: 3981\tFP: 2447\tTN: 7746\tFN: 1995\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233256889505228\n",
      "Acc: 0.53\n",
      "TP: 7\tFP: 4\tTN: 46\tFN: 43\n",
      "<2019-06-18 03:03:11.891439>  Epoch 99 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2250 - acc: 0.9032\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.997385012956258\n",
      "Acc: 0.7395015152452223\n",
      "TP: 3505\tFP: 1741\tTN: 8452\tFN: 2471\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233248893530927\n",
      "Acc: 0.53\n",
      "TP: 6\tFP: 3\tTN: 47\tFN: 44\n",
      "<2019-06-18 03:03:21.801915>  Epoch 100 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2133 - acc: 0.9081\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.020885595117758\n",
      "Acc: 0.738821201063764\n",
      "TP: 3562\tFP: 1809\tTN: 8384\tFN: 2414\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:03:31.664337>  Epoch 101 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2146 - acc: 0.9078\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.153329907827374\n",
      "Acc: 0.7349867029500897\n",
      "TP: 3610\tFP: 1919\tTN: 8274\tFN: 2366\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 03:03:41.506693>  Epoch 102 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2117 - acc: 0.9093\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.277235513611531\n",
      "Acc: 0.7313995918114912\n",
      "TP: 3778\tFP: 2145\tTN: 8048\tFN: 2198\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:03:51.344596>  Epoch 103 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2108 - acc: 0.9096\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.196063202262565\n",
      "Acc: 0.7337497680747108\n",
      "TP: 3814\tFP: 2143\tTN: 8050\tFN: 2162\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578628661505732\n",
      "Acc: 0.52\n",
      "TP: 4\tFP: 2\tTN: 48\tFN: 46\n",
      "<2019-06-18 03:04:01.240227>  Epoch 104 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2087 - acc: 0.9102\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.456682078599089\n",
      "Acc: 0.7262044653349001\n",
      "TP: 3962\tFP: 2413\tTN: 7780\tFN: 2014\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:04:11.091451>  Epoch 105 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2067 - acc: 0.9117\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.076426949941135\n",
      "Acc: 0.7372131857257715\n",
      "TP: 3586\tFP: 1859\tTN: 8334\tFN: 2390\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269420181352547\n",
      "Acc: 0.5\n",
      "TP: 4\tFP: 4\tTN: 46\tFN: 46\n",
      "<2019-06-18 03:04:20.988143>  Epoch 106 of 128\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 9s - loss: 0.2000 - acc: 0.9150\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.283643401100422\n",
      "Acc: 0.7312140515801843\n",
      "TP: 3766\tFP: 2136\tTN: 8057\tFN: 2210\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:04:30.816941>  Epoch 107 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.2008 - acc: 0.9150\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.253734486377557\n",
      "Acc: 0.7320799059929495\n",
      "TP: 3712\tFP: 2068\tTN: 8125\tFN: 2264\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924056405326347\n",
      "Acc: 0.51\n",
      "TP: 8\tFP: 7\tTN: 43\tFN: 42\n",
      "<2019-06-18 03:04:40.715995>  Epoch 108 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1974 - acc: 0.9145\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.783752316836026\n",
      "Acc: 0.7456861896221164\n",
      "TP: 3168\tFP: 1304\tTN: 8889\tFN: 2808\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:04:50.612218>  Epoch 109 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1959 - acc: 0.9163\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.225965836518261\n",
      "Acc: 0.7328839136619457\n",
      "TP: 3741\tFP: 2084\tTN: 8109\tFN: 2235\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578636657480033\n",
      "Acc: 0.52\n",
      "TP: 5\tFP: 3\tTN: 47\tFN: 45\n",
      "<2019-06-18 03:05:00.463556>  Epoch 110 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1946 - acc: 0.9176\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.208882883987636\n",
      "Acc: 0.7333786876120972\n",
      "TP: 3869\tFP: 2204\tTN: 7989\tFN: 2107\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269420181352547\n",
      "Acc: 0.5\n",
      "TP: 4\tFP: 4\tTN: 46\tFN: 46\n",
      "<2019-06-18 03:05:10.401722>  Epoch 111 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1903 - acc: 0.9198\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.266554860437592\n",
      "Acc: 0.7317088255303358\n",
      "TP: 3781\tFP: 2143\tTN: 8050\tFN: 2195\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924032417403442\n",
      "Acc: 0.51\n",
      "TP: 5\tFP: 4\tTN: 46\tFN: 45\n",
      "<2019-06-18 03:05:20.250076>  Epoch 112 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1927 - acc: 0.9187\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.099936186289677\n",
      "Acc: 0.7365328715443132\n",
      "TP: 3818\tFP: 2102\tTN: 8091\tFN: 2158\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269420181352547\n",
      "Acc: 0.5\n",
      "TP: 4\tFP: 4\tTN: 46\tFN: 46\n",
      "<2019-06-18 03:05:30.152246>  Epoch 113 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1902 - acc: 0.9195\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.87561932586819\n",
      "Acc: 0.7430267796400519\n",
      "TP: 3413\tFP: 1592\tTN: 8601\tFN: 2563\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924024421429138\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:05:40.061887>  Epoch 114 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1858 - acc: 0.9216\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.04225530839019\n",
      "Acc: 0.7382027336260746\n",
      "TP: 3726\tFP: 1983\tTN: 8210\tFN: 2250\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578652649428637\n",
      "Acc: 0.52\n",
      "TP: 7\tFP: 5\tTN: 45\tFN: 43\n",
      "<2019-06-18 03:05:49.898096>  Epoch 115 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1828 - acc: 0.9223\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.01234100334511\n",
      "Acc: 0.7390685880388398\n",
      "TP: 3563\tFP: 1806\tTN: 8387\tFN: 2413\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.233256889505228\n",
      "Acc: 0.53\n",
      "TP: 7\tFP: 4\tTN: 46\tFN: 43\n",
      "<2019-06-18 03:05:59.787174>  Epoch 116 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1791 - acc: 0.9254\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.582722608377097\n",
      "Acc: 0.7225555074525326\n",
      "TP: 4105\tFP: 2615\tTN: 7578\tFN: 1871\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.578644653454333\n",
      "Acc: 0.52\n",
      "TP: 6\tFP: 4\tTN: 46\tFN: 44\n",
      "<2019-06-18 03:06:09.643782>  Epoch 117 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1811 - acc: 0.9237\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.287920370247749\n",
      "Acc: 0.7310903580926464\n",
      "TP: 3860\tFP: 2232\tTN: 7961\tFN: 2116\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924048409352043\n",
      "Acc: 0.51\n",
      "TP: 7\tFP: 6\tTN: 44\tFN: 43\n",
      "<2019-06-18 03:06:19.556238>  Epoch 118 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1738 - acc: 0.9273\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.13838173093311\n",
      "Acc: 0.7354196301564723\n",
      "TP: 3710\tFP: 2012\tTN: 8181\tFN: 2266\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924040413377746\n",
      "Acc: 0.51\n",
      "TP: 6\tFP: 5\tTN: 45\tFN: 44\n",
      "<2019-06-18 03:06:29.450187>  Epoch 119 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1792 - acc: 0.9250\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.52504044471268\n",
      "Acc: 0.7242253695342941\n",
      "TP: 3987\tFP: 2470\tTN: 7723\tFN: 1989\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.26942817732685\n",
      "Acc: 0.5\n",
      "TP: 5\tFP: 5\tTN: 45\tFN: 45\n",
      "<2019-06-18 03:06:39.298265>  Epoch 120 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1721 - acc: 0.9277\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.157612217844418\n",
      "Acc: 0.7348630094625518\n",
      "TP: 3812\tFP: 2123\tTN: 8070\tFN: 2164\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:06:49.173028>  Epoch 121 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1695 - acc: 0.9293\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.281507982581598\n",
      "Acc: 0.7312758983239532\n",
      "TP: 3781\tFP: 2150\tTN: 8043\tFN: 2195\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.960203705225066\n",
      "Acc: 0.48\n",
      "TP: 3\tFP: 5\tTN: 45\tFN: 47\n",
      "<2019-06-18 03:06:59.022983>  Epoch 122 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1734 - acc: 0.9270\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.629718332925384\n",
      "Acc: 0.7211948790896159\n",
      "TP: 4109\tFP: 2641\tTN: 7552\tFN: 1867\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.269412185378247\n",
      "Acc: 0.5\n",
      "TP: 3\tFP: 3\tTN: 47\tFN: 47\n",
      "<2019-06-18 03:07:08.882404>  Epoch 123 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1706 - acc: 0.9298\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.839297924574176\n",
      "Acc: 0.7440781742841239\n",
      "TP: 3278\tFP: 1440\tTN: 8753\tFN: 2698\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.26942018135255\n",
      "Acc: 0.5\n",
      "TP: 4\tFP: 4\tTN: 46\tFN: 46\n",
      "<2019-06-18 03:07:18.784253>  Epoch 124 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1667 - acc: 0.9300\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.104200643955163\n",
      "Acc: 0.7364091780567753\n",
      "TP: 3659\tFP: 1945\tTN: 8248\tFN: 2317\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924016425454838\n",
      "Acc: 0.51\n",
      "TP: 3\tFP: 2\tTN: 48\tFN: 47\n",
      "<2019-06-18 03:07:28.633430>  Epoch 125 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1647 - acc: 0.9315\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.08497609134354\n",
      "Acc: 0.7369657987506958\n",
      "TP: 3677\tFP: 1954\tTN: 8239\tFN: 2299\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.924040413377746\n",
      "Acc: 0.51\n",
      "TP: 6\tFP: 5\tTN: 45\tFN: 44\n",
      "<2019-06-18 03:07:38.537059>  Epoch 126 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1610 - acc: 0.9345\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.149054966232443\n",
      "Acc: 0.7351103964376275\n",
      "TP: 3557\tFP: 1864\tTN: 8329\tFN: 2419\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 17.26942817732685\n",
      "Acc: 0.5\n",
      "TP: 5\tFP: 5\tTN: 45\tFN: 45\n",
      "<2019-06-18 03:07:48.387249>  Epoch 127 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1604 - acc: 0.9347\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 9.26228822686222\n",
      "Acc: 0.7318325190178737\n",
      "TP: 3896\tFP: 2256\tTN: 7937\tFN: 2080\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92402442142914\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "<2019-06-18 03:07:58.283658>  Epoch 128 of 128\n",
      "Epoch 1/1\n",
      " - 9s - loss: 0.1576 - acc: 0.9350\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.99737897975158\n",
      "Acc: 0.7395015152452223\n",
      "TP: 3383\tFP: 1619\tTN: 8574\tFN: 2593\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92402442142914\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n"
     ]
    }
   ],
   "source": [
    "model = build_model_LSTM_concat()\n",
    "\n",
    "bestEN = sys.float_info.max\n",
    "bestES = sys.float_info.max\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    printTrace('Epoch ' +str(epoch+1) + ' of ' +str(EPOCHS))\n",
    "    model.fit(train_set.X, train_set.y, batch_size=BATCH_SIZE, epochs=1, verbose=2)\n",
    "    print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "    \n",
    "    result_en = evaluate(model,dev_set)\n",
    "    \n",
    "    if result_en < bestEN:\n",
    "        bestEN = result_en\n",
    "        model.save(Save_path+'bestEN_LSTM_concat.h5')\n",
    "    \n",
    "    print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "    result_es = evaluate(model,test_set)\n",
    "    if result_es < bestES:\n",
    "        bestEN = result_en\n",
    "        model.save(Save_path+'bestES_LSTM_concat.h5')\n",
    "    \n",
    "    \n",
    "model.save(Save_path+'final_LSTM_concat.h5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ===== LAST: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.99737897975158\n",
      "Acc: 0.7395015152452223\n",
      "TP: 3383\tFP: 1619\tTN: 8574\tFN: 2593\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92402442142914\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "\n",
      "\n",
      "\n",
      " ===== BEST IN DEV SET: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.99737897975158\n",
      "Acc: 0.7395015152452223\n",
      "TP: 3383\tFP: 1619\tTN: 8574\tFN: 2593\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92402442142914\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n",
      "\n",
      "\n",
      "\n",
      " ===== BEST IN TEST SET: ====\n",
      "DEVELOPMENT SET (ENGLISH)\n",
      "Log Loss: 8.99737897975158\n",
      "Acc: 0.7395015152452223\n",
      "TP: 3383\tFP: 1619\tTN: 8574\tFN: 2593\n",
      "DEVELOPMENT SET (SPANISH)\n",
      "Log Loss: 16.92402442142914\n",
      "Acc: 0.51\n",
      "TP: 4\tFP: 3\tTN: 47\tFN: 46\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16.92402442142914"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\" ===== LAST: ====\")\n",
    "mode = load_model(Save_path+'final_LSTM_concat.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\" ===== BEST IN DEV SET: ====\")\n",
    "mode = load_model(Save_path+'bestEN_LSTM_concat.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\" ===== BEST IN TEST SET: ====\")\n",
    "mode = load_model(Save_path+'bestES_LSTM_concat.h5')\n",
    "print(\"DEVELOPMENT SET (ENGLISH)\")\n",
    "evaluate(model,dev_set)\n",
    "print(\"DEVELOPMENT SET (SPANISH)\")\n",
    "evaluate(model,test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}