Final Notebook with all models and results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Input, Dropout, Activation\n",
    "from keras.layers import Conv2D,GlobalMaxPooling2D\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.layers import Dense,merge,Lambda,Flatten\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping\n",
    "from keras.layers.merge import concatenate\n",
    "import keras.backend as K\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D  \n",
    "from keras.layers import Bidirectional, GlobalAveragePooling1D,GlobalMaxPool1D\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.layers import Dense,merge\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.utils import multi_gpu_model\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as plot\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "K.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 1732716473651071191\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 22684699853\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 12321255943730719310\n",
      "physical_device_desc: \"device: 0, name: Tesla M40 24GB, pci bus id: 0000:04:00.0, compute capability: 5.2\"\n",
      ", name: \"/device:GPU:1\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 22684699853\n",
      "locality {\n",
      "  bus_id: 2\n",
      "  numa_node: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 4840631246511674766\n",
      "physical_device_desc: \"device: 1, name: Tesla M40 24GB, pci bus id: 0000:82:00.0, compute capability: 5.2\"\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pat_df = pd.read_csv('patient_disease_vector.csv') #Patients\n",
    "\n",
    "with open('400PatComboArr25.pickle', 'rb') as handle :\n",
    "   combo = pickle.load(handle)\n",
    "\n",
    "with open('NewPatReportMatrix.pickle', 'rb') as handle :\n",
    "   new_vect = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SUBJECT_ID</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>249</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003799</td>\n",
       "      <td>0.189696</td>\n",
       "      <td>0.071951</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013072</td>\n",
       "      <td>0.275432</td>\n",
       "      <td>...</td>\n",
       "      <td>0.093745</td>\n",
       "      <td>0.091629</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.055828</td>\n",
       "      <td>0.026500</td>\n",
       "      <td>0.102564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.012821</td>\n",
       "      <td>0.217949</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.141026</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.064103</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.102564</td>\n",
       "      <td>0.217949</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>251</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>252</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.031746</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.070106</td>\n",
       "      <td>0.192857</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.194444</td>\n",
       "      <td>0.079894</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103439</td>\n",
       "      <td>0.046296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.115873</td>\n",
       "      <td>0.022487</td>\n",
       "      <td>0.070106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SUBJECT_ID    0         1         2         3         4  \\\n",
       "0           0         249  0.0  0.000000  0.003799  0.189696  0.071951   \n",
       "1           1         250  0.0  0.012821  0.217949  0.000000  0.153846   \n",
       "2           2         251  0.0  0.000000  0.000000  0.000000  0.000000   \n",
       "3           3         252  0.0  0.031746  0.000000  0.070106  0.192857   \n",
       "4           4         253  0.0  0.000000  0.000000  0.266667  0.000000   \n",
       "\n",
       "          5         6         7    ...            9        10   11   12   13  \\\n",
       "0  0.000000  0.013072  0.275432    ...     0.093745  0.091629  0.0  0.0  0.0   \n",
       "1  0.000000  0.000000  0.141026    ...     0.000000  0.064103  0.0  0.0  0.0   \n",
       "2  0.266667  0.000000  0.000000    ...     0.000000  0.000000  0.0  0.0  0.0   \n",
       "3  0.047619  0.194444  0.079894    ...     0.103439  0.046296  0.0  0.0  0.0   \n",
       "4  0.133333  0.111111  0.444444    ...     0.000000  0.000000  0.0  0.0  0.0   \n",
       "\n",
       "    14   15        16        17        18  \n",
       "0  0.0  0.0  0.055828  0.026500  0.102564  \n",
       "1  0.0  0.0  0.102564  0.217949  0.000000  \n",
       "2  0.0  0.0  0.000000  0.400000  0.333333  \n",
       "3  0.0  0.0  0.115873  0.022487  0.070106  \n",
       "4  0.0  0.0  0.000000  0.000000  0.000000  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pat_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_values(batch):\n",
    "    X1 = []\n",
    "    X2 = []\n",
    "    Y = np.empty((0,4), int)\n",
    "    for i in range(len(batch)):\n",
    "        p1, p2 = int(batch[i][0]) , int(batch[i][1])\n",
    "        X1.append(new_vect[p1])\n",
    "        X2.append(new_vect[p2])\n",
    "        vec1 = pat_df.loc[pat_df['SUBJECT_ID']==p1].iloc[0,8:12].values\n",
    "        vec2 = pat_df.loc[pat_df['SUBJECT_ID']==p2].iloc[0,8:12].values\n",
    "        y = np.logical_not(np.logical_xor(vec1,vec2)).astype(int)\n",
    "        Y = np.vstack((Y, y))\n",
    "    X1 = np.asarray(X1)\n",
    "    X2 = np.asarray(X2)\n",
    "    X1 = np.resize(X1,(X1.shape[0],X1.shape[1],X1.shape[2],1))\n",
    "    X2 = np.resize(X2,(X2.shape[0],X2.shape[1],X2.shape[2],1))\n",
    "    return [X1,X2], Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import Sequence\n",
    "\n",
    "class mimic_generator(Sequence):\n",
    "    \n",
    "    def __init__(self,combinations,batch_size):\n",
    "        self.combinations = combinations\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "    def __len__(self):\n",
    "        return int(len(self.combinations)/int(self.batch_size))\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        batch_x = self.combinations[index * self.batch_size :(index+1) * self.batch_size]\n",
    "        \n",
    "        \n",
    "        return  get_values(batch_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195761110\n"
     ]
    }
   ],
   "source": [
    "splitter = int(len(combo)*0.7)\n",
    "print(splitter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "combinations, X_test = combo[:splitter], combo[splitter:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "900000\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_17 (InputLayer)           (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_18 (InputLayer)           (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 300, 400, 64) 96064       input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 300, 400, 64) 96064       input_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling2d_17 (Global (None, 64)           0           conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling2d_18 (Global (None, 64)           0           conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)     (None, 128)          0           global_max_pooling2d_17[0][0]    \n",
      "                                                                 global_max_pooling2d_18[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 128)          16512       concatenate_8[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 4)            516         dense_15[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 209,156\n",
      "Trainable params: 209,156\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/5\n",
      "7030/7031 [============================>.] - ETA: 5s - loss: 0.3677 - acc: 0.8217 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-166:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 548, in _run\n",
      "    with closing(self.executor_fn(_SHARED_SEQUENCES)) as executor:\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 522, in <lambda>\n",
      "    initargs=(seqs,))\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/context.py\", line 118, in Pool\n",
      "    context=self.get_context())\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/pool.py\", line 174, in __init__\n",
      "    self._repopulate_pool()\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/pool.py\", line 239, in _repopulate_pool\n",
      "    w.start()\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/process.py\", line 105, in start\n",
      "    self._popen = self._Popen(self)\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/context.py\", line 267, in _Popen\n",
      "    return Popen(process_obj)\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/popen_fork.py\", line 20, in __init__\n",
      "    self._launch(process_obj)\n",
      "  File \"/home/nachiket/anaconda3/envs/myenv/lib/python3.5/multiprocessing/popen_fork.py\", line 67, in _launch\n",
      "    self.pid = os.fork()\n",
      "OSError: [Errno 12] Cannot allocate memory\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7031/7031 [==============================] - 43046s 6s/step - loss: 0.3677 - acc: 0.8217 - val_loss: 0.8260 - val_acc: 0.7676\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.76758, saving model to weights-CNN3.best.hdf5\n",
      "Epoch 2/5\n",
      " 189/7031 [..............................] - ETA: 11:11:33 - loss: 0.1357 - acc: 0.9523"
     ]
    }
   ],
   "source": [
    "combinations = combinations[:1000000]\n",
    "n = len(combinations)\n",
    "m = int(0.9 * n)\n",
    "print(m)\n",
    "dim = 300\n",
    "filter_size = 64\n",
    "ksize = 5\n",
    "num_training_samples = m\n",
    "num_validation_samples = n-m\n",
    "\n",
    "#Model1\n",
    "inp1 = Input(shape=(dim,400,1))\n",
    "x1 = Conv2D(filter_size, kernel_size=(dim,ksize) ,padding='same', kernel_initializer='normal',activation = 'relu')(inp1)\n",
    "\n",
    "x3 = GlobalMaxPooling2D()(x1)\n",
    "\n",
    "#Model2\n",
    "inp2 = Input(shape=(dim,400,1))\n",
    "y1 = Conv2D(filter_size, kernel_size=(dim,ksize) ,padding='same', kernel_initializer='normal', activation = 'relu')(inp2)\n",
    "y3 = GlobalMaxPooling2D()(y1)\n",
    "\n",
    "#Model3\n",
    "merged = concatenate([x3,y3])\n",
    "z1 = Dense(128, activation='relu')(merged)\n",
    "out = Dense(4, activation=\"sigmoid\")(z1)\n",
    "model = Model(inputs=[inp1,inp2], outputs=out)\n",
    "# try:\n",
    "#     model = multi_gpu_model(model)\n",
    "# except:\n",
    "#     print(\"Shit\")\n",
    "#adm = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "#Checkpoint\n",
    "filepath=\"weights-CNN3.best.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "#history=model.fit([X11,X222],Y1,validation_split=0.3, epochs=10, batch_size=64, callbacks=callbacks_list, verbose=1)\n",
    "my_training_batch_generator = mimic_generator(combinations[:m], batch_size=128)\n",
    "my_validation_batch_generator = mimic_generator(combinations[m:], batch_size=64)\n",
    "\n",
    "history = model.fit_generator(generator=my_training_batch_generator,\n",
    "                    steps_per_epoch=(num_training_samples // 128),\n",
    "                    epochs=5,\n",
    "                    verbose=1,\n",
    "                    validation_data=my_validation_batch_generator,\n",
    "                    validation_steps=(num_validation_samples // 64),\n",
    "                    use_multiprocessing=True,\n",
    "                    workers=5,\n",
    "                    max_queue_size=5,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "900000\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 1, 396, 64)   96064       input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 1, 396, 64)   96064       input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling2d_3 (GlobalM (None, 64)           0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling2d_4 (GlobalM (None, 64)           0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 128)          0           global_max_pooling2d_3[0][0]     \n",
      "                                                                 global_max_pooling2d_4[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 128)          16512       concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 4)            516         dense_3[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 209,156\n",
      "Trainable params: 209,156\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/5\n",
      "14062/14062 [==============================] - 3626s 258ms/step - loss: 3.0615 - acc: 0.6294 - val_loss: 3.0080 - val_acc: 0.6491\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.64908, saving model to weights-CNN.best.hdf5\n",
      "Epoch 2/5\n",
      "14062/14062 [==============================] - 3654s 260ms/step - loss: 3.0613 - acc: 0.6295 - val_loss: 3.0080 - val_acc: 0.6491\n",
      "\n",
      "Epoch 00002: val_acc did not improve from 0.64908\n",
      "Epoch 3/5\n",
      "14062/14062 [==============================] - 3651s 260ms/step - loss: 3.0613 - acc: 0.6295 - val_loss: 3.0080 - val_acc: 0.6491\n",
      "\n",
      "Epoch 00003: val_acc did not improve from 0.64908\n",
      "Epoch 4/5\n",
      "14062/14062 [==============================] - 3656s 260ms/step - loss: 3.0613 - acc: 0.6295 - val_loss: 3.0080 - val_acc: 0.6491\n",
      "\n",
      "Epoch 00004: val_acc did not improve from 0.64908\n",
      "Epoch 5/5\n",
      "14062/14062 [==============================] - 3676s 261ms/step - loss: 3.0613 - acc: 0.6295 - val_loss: 3.0080 - val_acc: 0.6491\n",
      "\n",
      "Epoch 00005: val_acc did not improve from 0.64908\n"
     ]
    }
   ],
   "source": [
    "combinations = combinations[:1000000]\n",
    "n = len(combinations)\n",
    "m = int(0.9 * n)\n",
    "print(m)\n",
    "dim = 300\n",
    "filter_size = 64\n",
    "ksize = 5\n",
    "num_training_samples = m\n",
    "num_validation_samples = n-m\n",
    "\n",
    "#Model1\n",
    "inp1 = Input(shape=(dim,400,1))\n",
    "x1 = Conv2D(filter_size, kernel_size=(dim,ksize) ,padding='valid', kernel_initializer='normal',activation = 'relu')(inp1)\n",
    "x2 = GlobalMaxPooling2D()(x1)\n",
    "\n",
    "#Model2\n",
    "inp2 = Input(shape=(dim,400,1))\n",
    "y1 = Conv2D(filter_size, kernel_size=(dim,ksize) ,padding='valid', kernel_initializer='normal', activation = 'relu')(inp2)\n",
    "y2 = GlobalMaxPooling2D()(y1)\n",
    "\n",
    "#Model3\n",
    "merged = concatenate([x2,y2])\n",
    "z1 = Dense(128, activation='relu')(merged)\n",
    "out = Dense(4, activation=\"sigmoid\")(z1)\n",
    "model = Model(inputs=[inp1,inp2], outputs=out)\n",
    "# try:\n",
    "#     model = multi_gpu_model(model)\n",
    "# except:\n",
    "#     print(\"Shit\")\n",
    "adm = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "#Checkpoint\n",
    "filepath=\"weights-CNN.best.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "#history=model.fit([X11,X222],Y1,validation_split=0.3, epochs=10, batch_size=64, callbacks=callbacks_list, verbose=1)\n",
    "my_training_batch_generator = mimic_generator(combinations[:m], batch_size=64)\n",
    "my_validation_batch_generator = mimic_generator(combinations[m:], batch_size=32)\n",
    "\n",
    "history = model.fit_generator(generator=my_training_batch_generator,\n",
    "                    steps_per_epoch=(num_training_samples // 64),\n",
    "                    epochs=5,\n",
    "                    verbose=1,\n",
    "                    validation_data=my_validation_batch_generator,\n",
    "                    validation_steps=(num_validation_samples // 32),\n",
    "                    use_multiprocessing=False,\n",
    "                    workers=5,\n",
    "                    max_queue_size=2,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'history' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-2e6bc19fd6fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# summarize history for accuracy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'model accuracy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'history' is not defined"
     ]
    }
   ],
   "source": [
    "# summarize history for accuracy\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "# summarize history for loss\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 1, 396, 64)   96064       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 1, 396, 64)   96064       input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling2d_1 (GlobalM (None, 64)           0           conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling2d_2 (GlobalM (None, 64)           0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 128)          0           global_max_pooling2d_1[0][0]     \n",
      "                                                                 global_max_pooling2d_2[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 128)          16512       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 4)            516         dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 209,156\n",
      "Trainable params: 209,156\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dim = 300\n",
    "filter_size = 64\n",
    "ksize = 5\n",
    "#Model1\n",
    "inp1 = Input(shape=(dim,400,1))\n",
    "x1 = Conv2D(filter_size, kernel_size=(dim,ksize) ,padding='valid', kernel_initializer='normal',activation = 'relu')(inp1)\n",
    "x2 = GlobalMaxPooling2D()(x1)\n",
    "\n",
    "#Model2\n",
    "inp2 = Input(shape=(dim,400,1))\n",
    "y1 = Conv2D(filter_size, kernel_size=(dim,ksize) ,padding='valid', kernel_initializer='normal', activation = 'relu')(inp2)\n",
    "y2 = GlobalMaxPooling2D()(y1)\n",
    "\n",
    "#Model3\n",
    "merged = concatenate([x2,y2])\n",
    "z1 = Dense(128, activation='relu')(merged)\n",
    "out = Dense(4, activation=\"sigmoid\")(z1)\n",
    "model = Model(inputs=[inp1,inp2], outputs=out)\n",
    "# try:\n",
    "#     model = multi_gpu_model(model)\n",
    "# except:\n",
    "#     print(\"Shit\")\n",
    "adm = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "#Checkpoint\n",
    "filepath=\"weights-CNN3.best.hdf5\"\n",
    "model.load_weights(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_sample_test = combinations[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_sample_test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-1c63a9f84ba3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_sample_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mp1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_sample_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_sample_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mX1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_vect\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mp1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X_sample_test' is not defined"
     ]
    }
   ],
   "source": [
    "X1 = []\n",
    "X2 = []\n",
    "Y = np.empty((0,4), int)\n",
    "\n",
    "for i in range(len(X_sample_test)):\n",
    "    p1, p2 = int(X_sample_test[i][0]) , int(X_sample_test[i][1])\n",
    "    X1.append(new_vect[p1])\n",
    "    X2.append(new_vect[p2])\n",
    "    vec1 = pat_df.loc[pat_df['SUBJECT_ID']==p1].iloc[0,8:12].values\n",
    "    vec2 = pat_df.loc[pat_df['SUBJECT_ID']==p2].iloc[0,8:12].values\n",
    "    y = np.logical_not(np.logical_xor(vec1,vec2)).astype(int)\n",
    "    Y = np.vstack((Y, y))\n",
    "X1 = np.asarray(X1)\n",
    "X2 = np.asarray(X2)\n",
    "X1 = np.resize(X1,(X1.shape[0],X1.shape[1],X1.shape[2],1))\n",
    "X2 = np.resize(X2,(X2.shape[0],X2.shape[1],X2.shape[2],1))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-5eeb90f0b0e5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate([X1,X2],Y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1.1408292901992798, 0.462475]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = [0.01 for i in range(4)]\n",
    "fp = [0.01 for i in range(4)]\n",
    "tn = [0.01 for i in range(4)]\n",
    "fn = [0.01 for i in range(4)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(scores)):\n",
    "    for j in range(4):\n",
    "        if ((scores[i][j] == Y[i][j]) & (Y[i][j] == 1)):\n",
    "            tp[j] += 1\n",
    "        if ((scores[i][j] == Y[i][j]) & (Y[i][j] == 0)):\n",
    "            tn[j] += 1\n",
    "        if ((scores[i][j] != Y[i][j]) & (Y[i][j] == 0)):\n",
    "            fp[j] += 1\n",
    "        if ((scores[i][j] != Y[i][j]) & (Y[i][j] == 1)):\n",
    "            fn[j] += 1\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diseases of nervous system and sense organs precision:85.46%, recall:72.35% and accuracy:55.77%\n",
      "Diseases of circulatory system precision:58.48%, recall:71.35% and accuracy:33.14%\n",
      "Diseases of respiratory system precision:31.74%, recall:43.55% and accuracy:53.94%\n",
      "Diseases of digestive precision:36.52%, recall:33.67% and accuracy:42.14%\n"
     ]
    }
   ],
   "source": [
    "print(\"Diseases of nervous system and sense organs precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[0]/(tp[0]+tn[0]))*100,(tp[0]/(tp[0]+fn[0]))*100,((tp[0]+tn[0])/(len(scores)))*100))\n",
    "print(\"Diseases of circulatory system precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[1]/(tp[1]+tn[1]))*100,(tp[1]/(tp[1]+fn[1]))*100,((tp[1]+tn[1])/(len(scores)))*100))\n",
    "print(\"Diseases of respiratory system precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[2]/(tp[2]+tn[2]))*100,(tp[2]/(tp[2]+fn[2]))*100,((tp[2]+tn[2])/(len(scores)))*100))\n",
    "print(\"Diseases of digestive precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[3]/(tp[3]+tn[3]))*100,(tp[3]/(tp[3]+fn[3]))*100,((tp[3]+tn[3])/(len(scores)))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 1. 0.]\n",
      " [1. 1. 1. 1.]\n",
      " [0. 1. 0. 0.]\n",
      " [1. 1. 1. 0.]\n",
      " [1. 1. 1. 0.]\n",
      " [1. 1. 1. 1.]\n",
      " [1. 1. 1. 0.]\n",
      " [0. 1. 0. 1.]\n",
      " [1. 1. 0. 0.]\n",
      " [0. 1. 1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "print(scores[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 1 1 1]\n",
      " [1 1 1 0]\n",
      " [0 0 0 1]\n",
      " [0 0 0 1]\n",
      " [1 0 0 1]\n",
      " [1 1 1 0]\n",
      " [0 1 0 0]\n",
      " [1 1 1 0]\n",
      " [1 1 0 1]\n",
      " [1 0 1 1]]\n"
     ]
    }
   ],
   "source": [
    "print(Y[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_values(batch):\n",
    "    X1 = []\n",
    "    X2 = []\n",
    "    Y = np.empty((0,4), int)\n",
    "    for i in range(len(batch)):\n",
    "        p1, p2 = int(batch[i][0]) , int(batch[i][1])\n",
    "        X1.append(new_vect[p1])\n",
    "        X2.append(new_vect[p2])\n",
    "        vec1 = pat_df.loc[pat_df['SUBJECT_ID']==p1].iloc[0,8:12].values\n",
    "        vec2 = pat_df.loc[pat_df['SUBJECT_ID']==p2].iloc[0,8:12].values\n",
    "        y = np.logical_not(np.logical_xor(vec1,vec2)).astype(int)\n",
    "        Y = np.vstack((Y, y))\n",
    "    X1 = np.asarray(X1)\n",
    "    X2 = np.asarray(X2)\n",
    "    return [X1,X2], Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Reshape, Flatten, LSTM, Dense, Dropout, Embedding, Bidirectional, GRU\n",
    "from keras.optimizers import Adam\n",
    "from keras import initializers, regularizers\n",
    "from keras import optimizers\n",
    "from keras.engine.topology import Layer\n",
    "from keras import constraints\n",
    "\n",
    "############################################## \n",
    "\"\"\"\n",
    "# ATTENTION LAYER\n",
    "Cite these works \n",
    "1. Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n",
    "\"Hierarchical Attention Networks for Document Classification\"\n",
    "accepted in NAACL 2016\n",
    "2. Winata, et al. https://arxiv.org/abs/1805.12307\n",
    "\"Attention-Based LSTM for Psychological Stress Detection from Spoken Language Using Distant Supervision.\" \n",
    "accepted in ICASSP 2018\n",
    "Using a context vector to assist the attention\n",
    "* How to use:\n",
    "Put return_sequences=True on the top of an RNN Layer (GRU/LSTM/SimpleRNN).\n",
    "The dimensions are inferred based on the output shape of the RNN.\n",
    "Example:\n",
    "\tmodel.add(LSTM(64, return_sequences=True))\n",
    "\tmodel.add(AttentionWithContext())\n",
    "\tmodel.add(Addition())\n",
    "\t# next add a Dense layer (for classification/regression) or whatever...\n",
    "\"\"\"\n",
    "##############################################\n",
    "\n",
    "def dot_product(x, kernel):\n",
    "\t\"\"\"\n",
    "\tWrapper for dot product operation, in order to be compatible with both\n",
    "\tTheano and Tensorflow\n",
    "\tArgs:\n",
    "\t\tx (): input\n",
    "\t\tkernel (): weights\n",
    "\tReturns:\n",
    "\t\"\"\"\n",
    "\tif K.backend() == 'tensorflow':\n",
    "\t\treturn K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n",
    "\telse:\n",
    "\t\treturn K.dot(x, kernel)\n",
    "\n",
    "class AttentionWithContext(Layer):\n",
    "\t\"\"\"\n",
    "\tAttention operation, with a context/query vector, for temporal data.\n",
    "\tSupports Masking.\n",
    "\tfollows these equations:\n",
    "\t\n",
    "\t(1) u_t = tanh(W h_t + b)\n",
    "\t(2) \\alpha_t = \\frac{exp(u^T u)}{\\sum_t(exp(u_t^T u))}, this is the attention weight\n",
    "\t(3) v_t = \\alpha_t * h_t, v in time t\n",
    "\t# Input shape\n",
    "\t\t3D tensor with shape: `(samples, steps, features)`.\n",
    "\t# Output shape\n",
    "\t\t3D tensor with shape: `(samples, steps, features)`.\n",
    "\t\"\"\"\n",
    "\n",
    "\tdef __init__(self,\n",
    "\t\t\t\t W_regularizer=None, u_regularizer=None, b_regularizer=None,\n",
    "\t\t\t\t W_constraint=None, u_constraint=None, b_constraint=None,\n",
    "\t\t\t\t bias=True, **kwargs):\n",
    "\n",
    "\t\tself.supports_masking = True\n",
    "\t\tself.init = initializers.get('glorot_uniform')\n",
    "\n",
    "\t\tself.W_regularizer = regularizers.get(W_regularizer)\n",
    "\t\tself.u_regularizer = regularizers.get(u_regularizer)\n",
    "\t\tself.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "\t\tself.W_constraint = constraints.get(W_constraint)\n",
    "\t\tself.u_constraint = constraints.get(u_constraint)\n",
    "\t\tself.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "\t\tself.bias = bias\n",
    "\t\tsuper(AttentionWithContext, self).__init__(**kwargs)\n",
    "\n",
    "\tdef build(self, input_shape):\n",
    "\t\tassert len(input_shape) == 3\n",
    "\n",
    "\t\tself.W = self.add_weight((input_shape[-1], input_shape[-1],),\n",
    "\t\t\t\t\t\t\t\t initializer=self.init,\n",
    "\t\t\t\t\t\t\t\t name='{}_W'.format(self.name),\n",
    "\t\t\t\t\t\t\t\t regularizer=self.W_regularizer,\n",
    "\t\t\t\t\t\t\t\t constraint=self.W_constraint)\n",
    "\t\tif self.bias:\n",
    "\t\t\tself.b = self.add_weight((input_shape[-1],),\n",
    "\t\t\t\t\t\t\t\t\t initializer='zero',\n",
    "\t\t\t\t\t\t\t\t\t name='{}_b'.format(self.name),\n",
    "\t\t\t\t\t\t\t\t\t regularizer=self.b_regularizer,\n",
    "\t\t\t\t\t\t\t\t\t constraint=self.b_constraint)\n",
    "\n",
    "\t\tself.u = self.add_weight((input_shape[-1],),\n",
    "\t\t\t\t\t\t\t\t initializer=self.init,\n",
    "\t\t\t\t\t\t\t\t name='{}_u'.format(self.name),\n",
    "\t\t\t\t\t\t\t\t regularizer=self.u_regularizer,\n",
    "\t\t\t\t\t\t\t\t constraint=self.u_constraint)\n",
    "\n",
    "\t\tsuper(AttentionWithContext, self).build(input_shape)\n",
    "\n",
    "\tdef compute_mask(self, input, input_mask=None):\n",
    "\t\t# do not pass the mask to the next layers\n",
    "\t\treturn None\n",
    "\n",
    "\tdef call(self, x, mask=None):\n",
    "\t\tuit = dot_product(x, self.W)\n",
    "\n",
    "\t\tif self.bias:\n",
    "\t\t\tuit += self.b\n",
    "\n",
    "\t\tuit = K.tanh(uit)\n",
    "\t\tait = dot_product(uit, self.u)\n",
    "\n",
    "\t\ta = K.exp(ait)\n",
    "\n",
    "\t\t# apply mask after the exp. will be re-normalized next\n",
    "\t\tif mask is not None:\n",
    "\t\t\t# Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "\t\t\ta *= K.cast(mask, K.floatx())\n",
    "\n",
    "\t\t# in some cases especially in the early stages of training the sum may be almost zero and this results in NaN's. \n",
    "\t\t# Should add a small epsilon as the workaround\n",
    "\t\t# a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n",
    "\t\ta /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "\t\ta = K.expand_dims(a)\n",
    "\t\tweighted_input = x * a\n",
    "\t\t\n",
    "\t\treturn weighted_input\n",
    "\n",
    "\tdef compute_output_shape(self, input_shape):\n",
    "\t\treturn input_shape[0], input_shape[1], input_shape[2]\n",
    "\t\n",
    "class Addition(Layer):\n",
    "\t\"\"\"\n",
    "\tThis layer is supposed to add of all activation weight.\n",
    "\tWe split this from AttentionWithContext to help us getting the activation weights\n",
    "\tfollows this equation:\n",
    "\t(1) v = \\sum_t(\\alpha_t * h_t)\n",
    "\t\n",
    "\t# Input shape\n",
    "\t\t3D tensor with shape: `(samples, steps, features)`.\n",
    "\t# Output shape\n",
    "\t\t2D tensor with shape: `(samples, features)`.\n",
    "\t\"\"\"\n",
    "\n",
    "\tdef __init__(self, **kwargs):\n",
    "\t\tsuper(Addition, self).__init__(**kwargs)\n",
    "\n",
    "\tdef build(self, input_shape):\n",
    "\t\tself.output_dim = input_shape[-1]\n",
    "\t\tsuper(Addition, self).build(input_shape)\n",
    "\n",
    "\tdef call(self, x):\n",
    "\t\treturn K.sum(x, axis=1)\n",
    "\n",
    "\tdef compute_output_shape(self, input_shape):\n",
    "\t\treturn (input_shape[0], self.output_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "950000\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 300, 400)     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            (None, 300, 400)     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_5 (CuDNNGRU)          (None, 300, 64)      89472       input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_6 (CuDNNGRU)          (None, 300, 64)      89472       input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "attention_with_context_5 (Atten (None, 300, 64)      4224        cu_dnngru_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "attention_with_context_6 (Atten (None, 300, 64)      4224        cu_dnngru_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "addition_5 (Addition)           (None, 64)           0           attention_with_context_5[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "addition_6 (Addition)           (None, 64)           0           attention_with_context_6[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 128)          0           addition_5[0][0]                 \n",
      "                                                                 addition_6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 256)          33024       concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 4)            1028        dense_5[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 221,444\n",
      "Trainable params: 221,444\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/1\n",
      "14843/14843 [==============================] - 4684s 316ms/step - loss: 0.3244 - acc: 0.8409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nachiket/anaconda3/envs/myenv/lib/python3.5/site-packages/keras/callbacks.py:432: RuntimeWarning: Can save best model only with val_acc available, skipping.\n",
      "  'skipping.' % (self.monitor), RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "combinations = combinations[:1000000]\n",
    "n = len(combinations)\n",
    "m = int(0.95 * n)\n",
    "print(m)\n",
    "dim = 300\n",
    "filter_size = 64\n",
    "num_training_samples = m\n",
    "num_validation_samples = n-m\n",
    "\n",
    "#model1\n",
    "inp1 = Input(shape=(dim,400))\n",
    "x2 = CuDNNGRU(64, return_sequences=True)(inp1)\n",
    "x3 = AttentionWithContext()(x2)\n",
    "x4 = Addition()(x3)\n",
    "\n",
    "#model2\n",
    "inp2 = Input(shape=(dim,400))\n",
    "z2 = CuDNNGRU(64, return_sequences=True)(inp2)\n",
    "z3 = AttentionWithContext()(z2)\n",
    "z4 = Addition()(z3)\n",
    "\n",
    "merged = concatenate([x4,z4])\n",
    "y1 = Dense(256, activation='relu')(merged)\n",
    "out = Dense(4, activation=\"sigmoid\")(y1)\n",
    "model = Model(inputs=[inp1,inp2], outputs=out)\n",
    "\n",
    "adm = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "#Checkpoint\n",
    "filepath=\"weights-GRU.best.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "# Fit model on training data\n",
    "#history=model.fit([X1,X2],Y1,validation_split=0.3, epochs=10, batch_size=64, callbacks=callbacks_list, verbose=1)\n",
    "my_training_batch_generator = mimic_generator(combo[:m], batch_size=64)\n",
    "my_validation_batch_generator = mimic_generator(combo[m:], batch_size=32)\n",
    "\n",
    "# model.fit_generator(generator=my_training_batch_generator,\n",
    "#                     steps_per_epoch=(num_training_samples // 64),\n",
    "#                     epochs=3,\n",
    "#                     verbose=1,\n",
    "#                     validation_data=my_validation_batch_generator,\n",
    "#                     validation_steps=(num_validation_samples // 64),\n",
    "#                     use_multiprocessing=True,\n",
    "#                     workers=3,\n",
    "#                     max_queue_size=14)\n",
    "history = model.fit_generator(generator=my_training_batch_generator,\n",
    "                    steps_per_epoch=(num_training_samples // 64),\n",
    "                    epochs=1,\n",
    "                    verbose=1,\n",
    "                    \n",
    "                    use_multiprocessing=True,\n",
    "                    workers=5,\n",
    "                    max_queue_size=14,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_sample_test = combo[splitter:splitter+10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1 = []\n",
    "X2 = []\n",
    "Y = np.empty((0,4), int)\n",
    "\n",
    "for i in range(len(X_sample_test)):\n",
    "    p1, p2 = int(X_sample_test[i][0]) , int(X_sample_test[i][1])\n",
    "    X1.append(new_vect[p1])\n",
    "    X2.append(new_vect[p2])\n",
    "    vec1 = pat_df.loc[pat_df['SUBJECT_ID']==p1].iloc[0,8:12].values\n",
    "    vec2 = pat_df.loc[pat_df['SUBJECT_ID']==p2].iloc[0,8:12].values\n",
    "    y = np.logical_not(np.logical_xor(vec1,vec2)).astype(int)\n",
    "    Y = np.vstack((Y, y))\n",
    "X1 = np.asarray(X1)\n",
    "X2 = np.asarray(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 21s 2ms/step\n",
      "[0.9989337594985962, 0.72815]\n"
     ]
    }
   ],
   "source": [
    "print(model.evaluate([X1,X2],Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.predict([X1,X2])\n",
    "for i in range(len(scores)):\n",
    "    for j in range(len(scores[i])):\n",
    "        if(scores[i][j] < 0.5):\n",
    "            scores[i][j] = 0\n",
    "        else:\n",
    "            scores[i][j] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = [0.01 for i in range(4)]\n",
    "fp = [0.01 for i in range(4)]\n",
    "tn = [0.01 for i in range(4)]\n",
    "fn = [0.01 for i in range(4)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(scores)):\n",
    "    for j in range(4):\n",
    "        if ((scores[i][j] == Y[i][j]) & (Y[i][j] == 1)):\n",
    "            tp[j] += 1\n",
    "        if ((scores[i][j] == Y[i][j]) & (Y[i][j] == 0)):\n",
    "            tn[j] += 1\n",
    "        if ((scores[i][j] != Y[i][j]) & (Y[i][j] == 0)):\n",
    "            fp[j] += 1\n",
    "        if ((scores[i][j] != Y[i][j]) & (Y[i][j] == 1)):\n",
    "            fn[j] += 1\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diseases of nervous system and sense organs precision:70.11%, recall:20.63% and accuracy:10.07%\n",
      "Diseases of circulatory system precision:75.80%, recall:99.64% and accuracy:96.16%\n",
      "Diseases of respiratory system precision:62.20%, recall:95.45% and accuracy:93.06%\n",
      "Diseases of digestive precision:46.09%, recall:92.82% and accuracy:91.97%\n"
     ]
    }
   ],
   "source": [
    "print(\"Diseases of nervous system and sense organs precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[0]/(tp[0]+tn[0]))*100,(tp[0]/(tp[0]+fn[0]))*100,((tp[0]+tn[0])/(len(scores)))*100))\n",
    "print(\"Diseases of circulatory system precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[1]/(tp[1]+tn[1]))*100,(tp[1]/(tp[1]+fn[1]))*100,((tp[1]+tn[1])/(len(scores)))*100))\n",
    "print(\"Diseases of respiratory system precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[2]/(tp[2]+tn[2]))*100,(tp[2]/(tp[2]+fn[2]))*100,((tp[2]+tn[2])/(len(scores)))*100))\n",
    "print(\"Diseases of digestive precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[3]/(tp[3]+tn[3]))*100,(tp[3]/(tp[3]+fn[3]))*100,((tp[3]+tn[3])/(len(scores)))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_values(batch):\n",
    "    X1 = []\n",
    "    X2 = []\n",
    "    Y = np.empty((0,4), int)\n",
    "    for i in range(len(batch)):\n",
    "        p1, p2 = int(batch[i][0]) , int(batch[i][1])\n",
    "        X1.append(new_vect[p1])\n",
    "        X2.append(new_vect[p2])\n",
    "        vec1 = pat_df.loc[pat_df['SUBJECT_ID']==p1].iloc[0,8:12].values\n",
    "        vec2 = pat_df.loc[pat_df['SUBJECT_ID']==p2].iloc[0,8:12].values\n",
    "        y = np.logical_not(np.logical_xor(vec1,vec2)).astype(int)\n",
    "        Y = np.vstack((Y, y))\n",
    "    X1 = np.asarray(X1)\n",
    "    X2 = np.asarray(X2)\n",
    "    X1 = np.resize(X1,(X1.shape[0],X1.shape[1],X1.shape[2],1))\n",
    "    X2 = np.resize(X2,(X2.shape[0],X2.shape[1],X2.shape[2],1))\n",
    "    return [X1,X2], Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nachiket/anaconda3/envs/myenv/lib/python3.5/site-packages/keras_applications/resnet50.py:263: UserWarning: The output shape of `ResNet50(include_top=False)` has been changed since Keras 2.2.0.\n",
      "  warnings.warn('The output shape of `ResNet50(include_top=False)` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            (None, 300, 400, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "resnet50 (Model)                multiple             23581440    input_2[0][0]                    \n",
      "                                                                 input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 266240)       0           resnet50[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 266240)       0           resnet50[2][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 532480)       0           flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          136315136   concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 4)            1028        dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 159,897,604\n",
      "Trainable params: 159,844,484\n",
      "Non-trainable params: 53,120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.resnet50 import ResNet50\n",
    "\n",
    "\n",
    "resnet50 = ResNet50(weights=None ,include_top = False,input_shape = (300,600,1))\n",
    "\n",
    "inp1 = Input(shape=(300,400,1))\n",
    "x1 = resnet50(inp1)\n",
    "x2 = Flatten()(x1)\n",
    "\n",
    "inp2 = Input(shape=(300,400,1))\n",
    "y1 = resnet50(inp2)\n",
    "y2 = Flatten()(y1)\n",
    "\n",
    "merged = concatenate([x2,y2])\n",
    "z1 = Dense(256, activation='relu')(merged)\n",
    "out = Dense(4, activation=\"softmax\")(z1)\n",
    "model = Model(inputs=[inp1,inp2], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "900\n",
      "Epoch 1/10\n",
      "14/14 [==============================] - 44s 3s/step - loss: 17.0130 - acc: 0.6652 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.79167, saving model to weights-ResCNN.best.hdf5\n",
      "Epoch 2/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 17.6292 - acc: 0.6920 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00002: val_acc did not improve from 0.79167\n",
      "Epoch 3/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 17.4133 - acc: 0.7009 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00003: val_acc did not improve from 0.79167\n",
      "Epoch 4/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 19.4281 - acc: 0.7589 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00004: val_acc did not improve from 0.79167\n",
      "Epoch 5/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 16.8377 - acc: 0.7589 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00005: val_acc did not improve from 0.79167\n",
      "Epoch 6/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 19.4281 - acc: 0.6875 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00006: val_acc did not improve from 0.79167\n",
      "Epoch 7/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 18.3487 - acc: 0.7143 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00007: val_acc did not improve from 0.79167\n",
      "Epoch 8/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 17.5572 - acc: 0.7188 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00008: val_acc did not improve from 0.79167\n",
      "Epoch 9/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 18.0609 - acc: 0.7009 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00009: val_acc did not improve from 0.79167\n",
      "Epoch 10/10\n",
      "14/14 [==============================] - 17s 1s/step - loss: 17.1255 - acc: 0.7188 - val_loss: 19.8118 - val_acc: 0.7917\n",
      "\n",
      "Epoch 00010: val_acc did not improve from 0.79167\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f7141d2a748>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combinations = combo[:1000000]\n",
    "n = len(combinations)\n",
    "m = int(0.90 * n)\n",
    "print(m)\n",
    "dim = 300\n",
    "filter_size = 64\n",
    "ksize = 5\n",
    "num_training_samples = m\n",
    "num_validation_samples = n-m\n",
    "\n",
    "#Checkpoint\n",
    "filepath=\"weights-ResCNN.best.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "#history=model.fit([X11,X222],Y1,validation_split=0.3, epochs=10, batch_size=64, callbacks=callbacks_list, verbose=1)\n",
    "my_training_batch_generator = mimic_generator(combinations[:m], batch_size=16)\n",
    "my_validation_batch_generator = mimic_generator(combinations[m:], batch_size=16)\n",
    "\n",
    "history = model.fit_generator(generator=my_training_batch_generator,\n",
    "                    steps_per_epoch=(num_training_samples // 64),\n",
    "                    epochs=10,\n",
    "                    verbose=1,\n",
    "                    validation_data=my_validation_batch_generator,\n",
    "                    validation_steps=(num_validation_samples // 32),\n",
    "                    use_multiprocessing=False,\n",
    "                    workers=5,\n",
    "                    max_queue_size=2,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'history' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-2e6bc19fd6fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# summarize history for accuracy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'model accuracy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'history' is not defined"
     ]
    }
   ],
   "source": [
    "# summarize history for accuracy\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "# summarize history for loss\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nachiket/anaconda3/envs/myenv/lib/python3.5/site-packages/keras_applications/resnet50.py:263: UserWarning: The output shape of `ResNet50(include_top=False)` has been changed since Keras 2.2.0.\n",
      "  warnings.warn('The output shape of `ResNet50(include_top=False)` '\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.resnet50 import ResNet50\n",
    "\n",
    "\n",
    "resnet50 = ResNet50(weights=None ,include_top = False,input_shape = (300,600,1))\n",
    "\n",
    "inp1 = Input(shape=(300,400,1))\n",
    "x1 = resnet50(inp1)\n",
    "x2 = Flatten()(x1)\n",
    "\n",
    "inp2 = Input(shape=(300,400,1))\n",
    "y1 = resnet50(inp2)\n",
    "y2 = Flatten()(y1)\n",
    "\n",
    "merged = concatenate([x2,y2])\n",
    "z1 = Dense(256, activation='relu')(merged)\n",
    "out = Dense(4, activation=\"softmax\")(z1)\n",
    "model = Model(inputs=[inp1,inp2], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.load_weights(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1 = []\n",
    "X2 = []\n",
    "Y = np.empty((0,4), int)\n",
    "\n",
    "for i in range(len(X_sample_test)):\n",
    "    p1, p2 = int(X_sample_test[i][0]) , int(X_sample_test[i][1])\n",
    "    X1.append(new_vect[p1])\n",
    "    X2.append(new_vect[p2])\n",
    "    vec1 = pat_df.loc[pat_df['SUBJECT_ID']==p1].iloc[0,8:12].values\n",
    "    vec2 = pat_df.loc[pat_df['SUBJECT_ID']==p2].iloc[0,8:12].values\n",
    "    y = np.logical_not(np.logical_xor(vec1,vec2)).astype(int)\n",
    "    Y = np.vstack((Y, y))\n",
    "X1 = np.asarray(X1)\n",
    "X2 = np.asarray(X2)\n",
    "X1 = np.resize(X1,(X1.shape[0],X1.shape[1],X1.shape[2],1))\n",
    "X2 = np.resize(X2,(X2.shape[0],X2.shape[1],X2.shape[2],1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.predict([X1,X2])\n",
    "for i in range(len(scores)):\n",
    "    for j in range(len(scores[i])):\n",
    "        if(scores[i][j] < 0.5):\n",
    "            scores[i][j] = 0\n",
    "        else:\n",
    "            scores[i][j] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = [0.01 for i in range(4)]\n",
    "fp = [0.01 for i in range(4)]\n",
    "tn = [0.01 for i in range(4)]\n",
    "fn = [0.01 for i in range(4)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(scores)):\n",
    "    for j in range(4):\n",
    "        if ((scores[i][j] == Y[i][j]) & (Y[i][j] == 1)):\n",
    "            tp[j] += 1\n",
    "        if ((scores[i][j] == Y[i][j]) & (Y[i][j] == 0)):\n",
    "            tn[j] += 1\n",
    "        if ((scores[i][j] != Y[i][j]) & (Y[i][j] == 0)):\n",
    "            fp[j] += 1\n",
    "        if ((scores[i][j] != Y[i][j]) & (Y[i][j] == 1)):\n",
    "            fn[j] += 1\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diseases of nervous system and sense organs precision:100.00%, recall:100.00% and accuracy:34.22%\n",
      "Diseases of circulatory system precision:0.00%, recall:0.00% and accuracy:26.85%\n",
      "Diseases of respiratory system precision:0.00%, recall:0.00% and accuracy:39.36%\n",
      "Diseases of digestive precision:0.00%, recall:0.00% and accuracy:54.33%\n"
     ]
    }
   ],
   "source": [
    "print(\"Diseases of nervous system and sense organs precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[0]/(tp[0]+tn[0]))*100,(tp[0]/(tp[0]+fn[0]))*100,((tp[0]+tn[0])/(len(scores)))*100))\n",
    "print(\"Diseases of circulatory system precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[1]/(tp[1]+tn[1]))*100,(tp[1]/(tp[1]+fn[1]))*100,((tp[1]+tn[1])/(len(scores)))*100))\n",
    "print(\"Diseases of respiratory system precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[2]/(tp[2]+tn[2]))*100,(tp[2]/(tp[2]+fn[2]))*100,((tp[2]+tn[2])/(len(scores)))*100))\n",
    "print(\"Diseases of digestive precision:{:.2f}%, recall:{:.2f}% and accuracy:{:.2f}%\".format((tp[3]/(tp[3]+tn[3]))*100,(tp[3]/(tp[3]+fn[3]))*100,((tp[3]+tn[3])/(len(scores)))*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}