Graph.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pickle\n",
    "import imageio\n",
    "import pandas as pd\n",
    "import keras\n",
    "import keras.losses\n",
    "import keras.backend as K\n",
    "from keras.models import load_model\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import json\n",
    "import seaborn as sn\n",
    "\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n",
    "plt.rcParams['figure.figsize'] = [13, 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LossHistory(keras.callbacks.Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.losses = []\n",
    "\n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.losses.append(logs.get('loss'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def matthews_correlation(y_true, y_pred):\n",
    "    '''Calculates the Matthews correlation coefficient measure for quality\n",
    "    of binary classification problems.\n",
    "    '''\n",
    "    '''  \n",
    "    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n",
    "    y_pred_neg = 1 - y_pred_pos\n",
    "\n",
    "    y_pos = K.round(K.clip(y_true, 0, 1))\n",
    "    y_neg = 1 - y_pos\n",
    "\n",
    "    tp = K.sum(y_pos * y_pred_pos)\n",
    "    tn = K.sum(y_neg * y_pred_neg)\n",
    "\n",
    "    fp = K.sum(y_neg * y_pred_pos)\n",
    "    fn = K.sum(y_pos * y_pred_neg)\n",
    "\n",
    "    numerator = (tp * tn - fp * fn)\n",
    "    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n",
    "\n",
    "    return numerator / (denominator + K.epsilon())\n",
    "    '''\n",
    "    return mcc(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history, filename):\n",
    "    loss_list = [s for s in history.keys() if 'loss' in s and 'val' not in s]\n",
    "    val_loss_list = [s for s in history.keys() if 'loss' in s and 'val' in s]\n",
    "    acc_list = [s for s in history.keys() if 'acc' in s and 'val' not in s]\n",
    "    val_acc_list = [s for s in history.keys() if 'acc' in s and 'val' in s]\n",
    "    f1_list = [s for s in history.keys() if 'f1' in s and 'val' not in s]\n",
    "    val_f1_list = [s for s in history.keys() if 'f1' in s and 'val' in s]\n",
    "    prec_list = [s for s in history.keys() if 'precision' in s and 'val' not in s]\n",
    "    val_prec_list = [s for s in history.keys() if 'precision' in s and 'val' in s]\n",
    "    rec_list = [s for s in history.keys() if 'recall' in s and 'val' not in s]\n",
    "    val_rec_list = [s for s in history.keys() if 'recall' in s and 'val' in s]\n",
    "\n",
    "    if len(loss_list) == 0:\n",
    "        print('Loss is missing in history')\n",
    "        return \n",
    "    \n",
    "    ## As loss always exists\n",
    "    epochs = range(1,len(history[loss_list[0]]) + 1)\n",
    "    \n",
    "    ## Loss\n",
    "    #plt.figure(1)\n",
    "    plt.subplot(5, 2, 1)\n",
    "    for l in loss_list:\n",
    "        plt.plot(epochs, history[l], 'b', label='Training loss (' + str(str(format(history[l][-1],'.5f'))+')'))\n",
    "    \n",
    "    plt.title('Training Loss')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Training Loss')\n",
    "    plt.legend()\n",
    "    \n",
    "    plt.subplot(5,2,2)\n",
    "    for l in val_loss_list:\n",
    "        plt.plot(epochs, history[l], 'g', label='Validation loss (' + str(str(format(history[l][-1],'.5f'))+')'))\n",
    "    \n",
    "    plt.title('Validation Loss')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Validation Loss')\n",
    "    plt.legend()\n",
    "    #plt.savefig(filename)\n",
    "    plt.ylim(0, 1)\n",
    "\n",
    "    ## Accuracy\n",
    "    if(len(acc_list) != 0):\n",
    "        #plt.figure(2)\n",
    "        plt.subplot(5, 2, 3)\n",
    "        for l in acc_list:\n",
    "            plt.plot(epochs, history[l], 'b', label='Training accuracy (' + str(format(history[l][-1],'.5f'))+')')\n",
    "            \n",
    "        plt.title('Training Accuracy')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training Accuracy')\n",
    "        plt.legend()\n",
    "        plt.ylim(0,1)\n",
    "        #plt.savefig(filename)\n",
    "\n",
    "        plt.subplot(5 ,2, 4)\n",
    "        for l in val_acc_list:    \n",
    "            plt.plot(epochs, history[l], 'g', label='Validation accuracy (' + str(format(history[l][-1],'.5f'))+')')\n",
    "\n",
    "        plt.title('Validation Accuracy')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Validation Accuracy')\n",
    "        plt.legend()\n",
    "        plt.ylim(0,1)\n",
    "        #plt.savefig(filename)\n",
    "\n",
    "    ## F1\n",
    "    if(len(f1_list) != 0):\n",
    "        #plt.figure(2)\n",
    "        plt.subplot(5, 2, 5)\n",
    "        for l in f1_list:\n",
    "            plt.plot(epochs, history[l], 'b', label='Training F1 (' + str(format(history[l][-1],'.5f'))+')')\n",
    "        plt.title('Training F1 Score')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training F1 Score')\n",
    "        plt.legend()\n",
    "        #plt.savefig(filename)  \n",
    "        \n",
    "        plt.subplot(5, 2, 6)\n",
    "        for l in val_f1_list:    \n",
    "            plt.plot(epochs, history[l], 'g', label='Validation F1 (' + str(format(history[l][-1],'.5f'))+')')\n",
    "\n",
    "        plt.title('Validation F1 Score')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Validation F1 Score')\n",
    "        plt.legend()\n",
    "        #plt.savefig(filename)\n",
    "        \n",
    "    ## Precision\n",
    "    if(len(prec_list) != 0):\n",
    "        #plt.figure(2)\n",
    "        plt.subplot(5, 2, 7)\n",
    "        for l in prec_list:\n",
    "            plt.plot(epochs, history[l], 'b', label='Training Precision (' + str(format(history[l][-1],'.5f'))+')')\n",
    "        plt.title('Training Precision')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training Precision')\n",
    "        plt.legend()\n",
    "        #plt.savefig(filename)\n",
    "        \n",
    "        plt.subplot(5, 2, 8)\n",
    "        for l in val_prec_list:    \n",
    "            plt.plot(epochs, history[l], 'g', label='Validation Precision (' + str(format(history[l][-1],'.5f'))+')')\n",
    "\n",
    "        plt.title('Validation Precision')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Validation Precision')\n",
    "        plt.legend()\n",
    "        #plt.savefig(filename)\n",
    "\n",
    "    ## Recall\n",
    "    if(len(rec_list) != 0):\n",
    "        #plt.figure(2)\n",
    "        plt.subplot(5, 2, 9)\n",
    "        for l in rec_list:\n",
    "            plt.plot(epochs, history[l], 'b', label='Training Recall (' + str(format(history[l][-1],'.5f'))+')')\n",
    "        plt.title('Training Recall')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training Recall')\n",
    "        plt.legend()\n",
    "        #plt.savefig(filename)\n",
    "\n",
    "        \n",
    "        plt.subplot(5, 2, 10)\n",
    "        for l in val_rec_list:    \n",
    "            plt.plot(epochs, history[l], 'g', label='Validation Recall (' + str(format(history[l][-1],'.5f'))+')')\n",
    "\n",
    "        plt.title('Validation Recall')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Validation Recall')\n",
    "        plt.legend()\n",
    "        #plt.savefig(filename)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = \"./Models/JSON/tests\"\n",
    "fileName = \"noKFold3Class_lr-5.json\"\n",
    "f = plt.figure(figsize=(13,13))\n",
    "history = json.load(open(os.path.join(dir, fileName) ,\"r\"))\n",
    "saveTo = os.path.join(\"./Graphs\", (fileName[:-5]+\".png\"))\n",
    "plot_history(history, saveTo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model(\"./Models/model/tests/noKFold3Class_lr-5.h5\", \n",
    "                   custom_objects={\"precision\": precision_m, \n",
    "                                   \"recall_m\": recall_m}) \n",
    "plot_model(model, to_file='./Graphs/model_vis.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model(\"./Models/model/tests/noKFold3Class_lr-5.h5\", \n",
    "                   custom_objects={\"f1_m\": f1_m,\n",
    "                                   \"precision_m\": precision_m, \n",
    "                                   \"recall_m\": recall_m}) \n",
    "                   #, custom_objects={'matthews_correlation': matthews_correlation})\n",
    "X = []\n",
    "y = []\n",
    "noPlaqueErrors = []\n",
    "noPlaqueErrorsPred = []\n",
    "calErrors = []\n",
    "calErrorsPred = []\n",
    "fibErrors = []\n",
    "fibErrorsPred = []\n",
    "errors = 0\n",
    "\n",
    "dir = \"./Data/Patches/NoPlaque\"\n",
    "for filename in os.listdir(dir):\n",
    "    img = np.array(imageio.imread(os.path.join(dir, filename)))[:,:,0:1]\n",
    "    #img = np.load(os.path.join(dir, filename))[:,:,0]\n",
    "    X.append(img)\n",
    "    y.append([1,0,0])\n",
    "    val = np.expand_dims(img, axis=0)\n",
    "    if np.argmax(model.predict(val)) != 0:\n",
    "        errors += 1\n",
    "        noPlaqueErrors.append(img)\n",
    "        noPlaqueErrorsPred.append(model.predict(val))\n",
    "\n",
    "#X = np.array(X)\n",
    "#y = np.array(y)\n",
    "#for i in range(20):\n",
    "#    print(y[i])\n",
    "#    val = np.expand_dims(X[i][:,:,0:1], axis=0)\n",
    "#    print(np.argmax(model.predict(val)))\n",
    "#    plt.imshow(X[i]/256)\n",
    "#    plt.show()\n",
    "\n",
    "dir = \"./Data/Patches/Plaque/Aug/cal\"\n",
    "for filename in os.listdir(dir):\n",
    "    img = np.array(imageio.imread(os.path.join(dir, filename)))[:,:,0:1]\n",
    "    #img = np.load(os.path.join(dir, filename))[:,:,0]\n",
    "    X.append(img)\n",
    "    y.append([0,1,0])\n",
    "    val = np.expand_dims(img, axis=0)\n",
    "    if np.argmax(model.predict(val)) != 1:\n",
    "        errors += 1\n",
    "        calErrors.append(img)\n",
    "        calErrorsPred.append(model.predict(val))\n",
    "    \n",
    "#X = np.array(X)\n",
    "#y = np.array(y)\n",
    "#for i in range(20):\n",
    "#    print(y[i])\n",
    "#    val = np.expand_dims(X[i][:,:,0:1], axis=0)\n",
    "#    print(np.argmax(model.predict(val)))\n",
    "#    plt.imshow(X[i]/256)\n",
    "#    plt.show()\n",
    "\n",
    "dir = \"./Data/Patches/Plaque/Aug/fibrous\"\n",
    "for filename in os.listdir(dir):\n",
    "    img = np.array(imageio.imread(os.path.join(dir, filename)))[:,:,0:1]\n",
    "    #img = np.load(os.path.join(dir, filename))[:,:,0]\n",
    "    X.append(img)\n",
    "    y.append([0,0,1])\n",
    "    val = np.expand_dims(img, axis=0)\n",
    "    if np.argmax(model.predict(val)) != 2:\n",
    "        errors += 1\n",
    "        fibErrors.append(img)\n",
    "        fibErrorsPred.append(model.predict(val))\n",
    "#X = np.array(X)\n",
    "#y = np.array(y)\n",
    "#for i in range(20):\n",
    "#    print(y[i])\n",
    "#    val = np.expand_dims(X[i][:,:,0:1], axis=0)\n",
    "#    print(np.argmax(model.predict(val)))\n",
    "#    plt.imshow(X[i]/256)\n",
    "#    plt.show()\n",
    "\n",
    "X = np.array(X)\n",
    "y = np.array(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(noPlaqueErrors[5][:,:,0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = \"./Data/Patches/NoPlaque\"\n",
    "filename = os.listdir(dir)[0]\n",
    "img = np.array(imageio.imread(os.path.join(dir, filename)))[:,:,0]\n",
    "plt.imshow(img[:,:])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(errors/len(X))*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(X)\n",
    "cmatrix = np.zeros((3,3))\n",
    "for i,y_our in enumerate(y_pred):\n",
    "    cmatrix[np.argmax(y_our)][np.argmax(y[i])] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cm = pd.DataFrame(cmatrix, index = [\"Normal\", \"Cal\", \"Fib\"], columns = [\"Normal\", \"Cal\", \"Fib\"])\n",
    "plt.figure(figsize = (10,7))\n",
    "snheatmap = sn.heatmap(df_cm, annot=True,  fmt='g', cmap=\"Blues\")\n",
    "snheatmap.get_figure().savefig(\"Graphs/CONFUSION_MATRIX_1_KFOLD_TEST.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cm = pd.DataFrame(cmatrix[1:,1:], index = [\"Cal\", \"Fib\"], columns = [\"Cal\", \"Fib\"])\n",
    "plt.figure(figsize = (10,7))\n",
    "snheatmap = sn.heatmap(df_cm, annot=True,  fmt='g', cmap=\"Blues\")\n",
    "snheatmap.get_figure().savefig(\"Graphs/CONFUSION_MATRIX_2_TEST.png\")\n",
    "\n",
    "print(\"Acc plaque detection: \", 1-(cmatrix[1:,1:][0][1] + cmatrix[1:,1:][1][0])/np.sum(cmatrix[1:,1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmatrix = np.zeros((2,2))\n",
    "for i,y_our in enumerate(y_pred):\n",
    "    val = min(1, np.argmax(y_our))\n",
    "    val1 = min(1, np.argmax(y[i]))\n",
    "    cmatrix[val][val1] += 1\n",
    "df_cm = pd.DataFrame(cmatrix, index = [\"Normal\", \"Plaque\"], columns = [\"Normal\", \"Plaque\"])\n",
    "plt.figure(figsize = (10,7))\n",
    "snheatmap = sn.heatmap(df_cm, annot=True,  fmt='g', cmap=\"Blues\")\n",
    "snheatmap.get_figure().savefig(\"Graphs/CONFUSION_MATRIX_3_KFOLD_TEST_2CLASS.png\")\n",
    "\n",
    "print(\"Acc plaque detection: \", 1-(cmatrix[0][1] + cmatrix[1][0])/np.sum(cmatrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}