plot_predictions_and_get_scores.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ISBI Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scoring Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "\n",
    "# Score Utilities Imports\n",
    "from code.scoring.utilities import scoring, dense_scoring, to_strange_fmt, resize_keypoints_to_original_size\n",
    "\n",
    "\n",
    "# GLOBAL VARIABLES\n",
    "FOLDS = [i for i in range(5)]\n",
    "\n",
    "# Directories\n",
    "# Data\n",
    "data_dir = 'data'\n",
    "\n",
    "# Resized Data\n",
    "resized_dir = os.path.join(data_dir, 'resized')\n",
    "\n",
    "# Original Data\n",
    "original_dir = os.path.join(data_dir, 'original')\n",
    "\n",
    "# Train and Test Indices\n",
    "test_indices_list_path = os.path.join(data_dir, 'train-test-indices', 'test_indices_list.pickle')\n",
    "\n",
    "# ISBI Results Directories\n",
    "isbi_results_dir = os.path.join('results', 'isbi-model', 'predictions')\n",
    "\n",
    "\n",
    "# Go trough folds\n",
    "for fold in FOLDS:\n",
    "    print('Current fold {}'.format(fold))\n",
    "\n",
    "    # Open test indices list\n",
    "    with open(test_indices_list_path, 'rb') as t:\n",
    "        test_indices_list = cPickle.load(t)\n",
    "\n",
    "    # Open predictions files\n",
    "    with open(os.path.join(isbi_results_dir, 'isbi_preds_w_only_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Keypoints were normalized by the image width (512), so we need to \"denormalize\" them\n",
    "    pred_kpts = np.array(pred_kpts)\n",
    "    pred_kpts *= 512\n",
    "\n",
    "\n",
    "    # print(pred_kpts)\n",
    "\n",
    "    # Open X data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'X_train_221.pickle'), 'rb') as fp: \n",
    "        X_train_original = cPickle.load(fp) \n",
    "\n",
    "    # Test\n",
    "    with open(os.path.join(original_dir, 'X_test_221.pickle'),'rb') as fp: \n",
    "        X_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain complete dataset\n",
    "    X_original = np.concatenate((X_train_original, X_test_original))\n",
    "\n",
    "    # Obtain the images list related to the fold\n",
    "    X_original = X_original[test_indices_list[fold]]\n",
    "\n",
    "    # Resized predictions to the original size\n",
    "    print('Resizing ISBI predictions to original sizes...')\n",
    "    resized_preds = resize_keypoints_to_original_size(pred_kpts, X_original.copy())\n",
    "    print('ISBI predictions resized.')\n",
    "\n",
    "\n",
    "    # Open y data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'y_train_221.pickle'), 'rb') as fp:\n",
    "        y_train_original = cPickle.load(fp)  \n",
    "\n",
    "    with open(os.path.join(original_dir, 'y_test_221.pickle'), 'rb') as fp:  \n",
    "        y_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain the complete dataset     \n",
    "    y_original = np.concatenate((y_train_original, y_test_original))\n",
    "    \n",
    "    # Obtain the keypoints related to the fold\n",
    "    y_original = y_original[test_indices_list[fold]]\n",
    "\n",
    "\n",
    "    # Create a list append the processing stuff\n",
    "    print('Creating a list to process...')\n",
    "    predictions = [] \n",
    "\n",
    "    # Go through all the images in this fold\n",
    "    for i in range(np.shape(X_original)[0]):\n",
    "        predictions.append(to_strange_fmt(resized_preds[i]))\n",
    "\n",
    "\n",
    "    # Create a scores list to process\n",
    "    print('Creating a scores list to process...')\n",
    "    scores = []\n",
    "\n",
    "    # Go through all the images in this fold\n",
    "    for i in range(np.shape(X_original)[0]): \n",
    "        scores.append(scoring(predictions=predictions[i], y=y_original[i], img_shape=X_original[i].shape, dataset_diagonal=2701.6085, dataset=\"221\"))\n",
    "    \n",
    "    # Generating dense scores\n",
    "    print('Generating final dense scores...')\n",
    "    dense_scores = dense_scoring(scores)\n",
    "\n",
    "    # Print scores\n",
    "    print('MODEL: {} | FOLD: {}'.format('ISBI', fold))\n",
    "    print('ENDPOINTS [MEAN, STD, MAX]: {}'.format(dense_scores[0]))\n",
    "    print('BREAST CONTOUR [MEAN, STD, MAX]: {}'.format(dense_scores[1]))\n",
    "    print('NIPPLES [MEAN, STD, MAX]: {}\\n'.format(dense_scores[2]))\n",
    "\n",
    "# Finish statement\n",
    "print(\"ISBI Model Scoring Results finished.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "\n",
    "# Score Utilities Imports\n",
    "from code.scoring.utilities import scoring, dense_scoring, to_strange_fmt, resize_keypoints_to_original_size\n",
    "\n",
    "\n",
    "# GLOBAL VARIABLES\n",
    "FOLDS = [i for i in range(5)]\n",
    "\n",
    "# Directories\n",
    "# Data\n",
    "data_dir = 'data'\n",
    "\n",
    "# Resized Data\n",
    "resized_dir = os.path.join(data_dir, 'resized')\n",
    "\n",
    "# Original Data\n",
    "original_dir = os.path.join(data_dir, 'original')\n",
    "\n",
    "# Train and Test Indices\n",
    "test_indices_list_path = os.path.join(data_dir, 'train-test-indices', 'test_indices_list.pickle')\n",
    "\n",
    "# ISBI Results Directories\n",
    "isbi_results_dir = os.path.join('results', 'isbi-model', 'predictions')\n",
    "\n",
    "\n",
    "# Go trough folds\n",
    "for fold in FOLDS:\n",
    "    print('Current fold {}'.format(fold))\n",
    "\n",
    "    # Open test indices list\n",
    "    with open(test_indices_list_path, 'rb') as t:\n",
    "        test_indices_list = cPickle.load(t)\n",
    "\n",
    "    # Open predictions files\n",
    "    with open(os.path.join(isbi_results_dir, 'isbi_preds_w_only_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Keypoints were normalized by the image width (512), so we need to \"denormalize\" them\n",
    "    pred_kpts = np.array(pred_kpts)\n",
    "    pred_kpts *= 512\n",
    "\n",
    "\n",
    "    # print(pred_kpts)\n",
    "\n",
    "    # Open X data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'X_train_221.pickle'), 'rb') as fp: \n",
    "        X_train_original = cPickle.load(fp) \n",
    "\n",
    "    # Test\n",
    "    with open(os.path.join(original_dir, 'X_test_221.pickle'),'rb') as fp: \n",
    "        X_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain complete dataset\n",
    "    X_original = np.concatenate((X_train_original, X_test_original))\n",
    "\n",
    "    # Obtain the images list related to the fold\n",
    "    X_original = X_original[test_indices_list[fold]]\n",
    "\n",
    "    # Resized predictions to the original size\n",
    "    print('Resizing ISBI predictions to original sizes...')\n",
    "    resized_preds = resize_keypoints_to_original_size(pred_kpts, X_original.copy())\n",
    "    print('ISBI predictions resized.')\n",
    "\n",
    "\n",
    "    # Open y data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'y_train_221.pickle'), 'rb') as fp:\n",
    "        y_train_original = cPickle.load(fp)  \n",
    "\n",
    "    with open(os.path.join(original_dir, 'y_test_221.pickle'), 'rb') as fp:  \n",
    "        y_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain the complete dataset     \n",
    "    y_original = np.concatenate((y_train_original, y_test_original))\n",
    "    \n",
    "    # Obtain the keypoints related to the fold\n",
    "    y_original = y_original[test_indices_list[fold]]\n",
    "    \n",
    "    \n",
    "    # Generate Plots\n",
    "    for index, image in enumerate(X_original):\n",
    "        plt.title('ISBI Plots | Fold {} of {} | Image {} of {}'.format(fold, len(FOLDS)-1, index+1, len(images_list[test_indices_list[fold]])))\n",
    "        plt.imshow(image, cmap='gray')\n",
    "        plt.plot(y_original[index][0:74:2], y_original[index][1:75:2], 'bo')\n",
    "        plt.plot(resized_preds[index][0:74:2], resized_preds[index][1:75:2], 'ro')\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hybrid Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scoring Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "\n",
    "# Score Utilities Imports\n",
    "from code.scoring.utilities import scoring, dense_scoring, to_strange_fmt, resize_keypoints_to_original_size\n",
    "\n",
    "\n",
    "# GLOBAL VARIABLES\n",
    "FOLDS = [i for i in range(5)]\n",
    "\n",
    "# Directories\n",
    "# Data\n",
    "data_dir = 'data'\n",
    "\n",
    "# Resized Data\n",
    "resized_dir = os.path.join(data_dir, 'resized')\n",
    "\n",
    "# Original Data\n",
    "original_dir = os.path.join(data_dir, 'original')\n",
    "\n",
    "# Train and Test Indices\n",
    "test_indices_list_path = os.path.join(data_dir, 'train-test-indices', 'test_indices_list.pickle')\n",
    "\n",
    "# ISBI Results Directories\n",
    "isbi_results_dir = os.path.join('results', 'isbi-model', 'predictions')\n",
    "\n",
    "# Hybrid Results Directories\n",
    "hybrid_results_dir = os.path.join('results', 'hybrid-model', 'predictions', 'reshaped')\n",
    "\n",
    "\n",
    "# Go trough folds\n",
    "for fold in FOLDS:\n",
    "    print('Current fold {}'.format(fold))\n",
    "\n",
    "    # Open test indices list\n",
    "    with open(test_indices_list_path, 'rb') as t:\n",
    "        test_indices_list = cPickle.load(t)\n",
    "\n",
    "    # Open predictions files\n",
    "    # ISBI Predictions\n",
    "    with open(os.path.join(isbi_results_dir, 'isbi_preds_w_only_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        isbi_pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Keypoints were normalized by the image width (512), so we need to \"denormalize\" them\n",
    "    isbi_pred_kpts = np.array(isbi_pred_kpts)\n",
    "    isbi_pred_kpts *= 512\n",
    "\n",
    "\n",
    "    # Hybrid Predictions\n",
    "    with open(os.path.join(hybrid_results_dir, 'hybrid_preds_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        hybrid_pred_kpts = cPickle.load(f)\n",
    "\n",
    "    # print(hybrid_pred_kpts)\n",
    "\n",
    "    # Convert to NumPy array\n",
    "    hybrid_pred_kpts = np.array(hybrid_pred_kpts)\n",
    "    # print(hybrid_predictions.shape)\n",
    "    \n",
    "    # We have to be sure that the Hybrid Predictions of Endpoints and Nipples are the same as ISBI's\n",
    "    # Endpoints\n",
    "    # truth_lp = y[0:2]\n",
    "    # truth_midl = y[32:34]\n",
    "    # truth_midr = y[66:68]\n",
    "    # truth_rp = y[34:36]\n",
    "    \n",
    "    # Nipples\n",
    "    # truth_l_nipple = y[70:72]\n",
    "    # truth_r_nipple = y[72:74]\n",
    "    \n",
    "    # Begin Process Statement\n",
    "    print('Updating Hybrid Model Endpoints for scoring purposes...')\n",
    "    \n",
    "    # Iterate through ISBI predictions and update Hybrid Predictions Accordingly\n",
    "    for index, value in enumerate(isbi_pred_kpts):\n",
    "        # Endpoints\n",
    "        hybrid_pred_kpts[index][0:2] = value.copy()[0:2]\n",
    "        # hybrid_predictions[index][0:2] = np.round(hybrid_predictions[index][0:2], 5)\n",
    "        # hybrid_predictions[index][1] = value[1]\n",
    "        \n",
    "        hybrid_pred_kpts[index][32:34] = value.copy()[32:34]\n",
    "        # hybrid_predictions[index][32:34] = np.round(hybrid_predictions[index][32:34], 5)\n",
    "        # hybrid_predictions[index][33] = value[33]\n",
    "        \n",
    "        # Mid-points are inversed in hybrid predictions\n",
    "        hybrid_pred_kpts[index][66:68] = value.copy()[66:68]\n",
    "        # hybrid_predictions[index][66:68] = np.round(hybrid_predictions[index][66:68], 5)\n",
    "        # hybrid_predictions[index][67] = value[35]\n",
    "        \n",
    "        hybrid_pred_kpts[index][34:36] = value.copy()[34:36]\n",
    "        # hybrid_predictions[index][34:36] = np.round(hybrid_predictions[index][34:36], 5)\n",
    "        # hybrid_predictions[index][35] = value[67]\n",
    "        \n",
    "        # Nipples\n",
    "        hybrid_pred_kpts[index][70:72] = value.copy()[70:72]\n",
    "        # hybrid_predictions[index][71] = value[71]\n",
    "        hybrid_pred_kpts[index][72:74] = value.copy()[72:74]\n",
    "        # hybrid_predictions[index][73] = value[73] \n",
    "    \n",
    "    print('Hybrid Model Endpoints updated.')\n",
    "\n",
    "\n",
    "    # Open X data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'X_train_221.pickle'), 'rb') as fp: \n",
    "        X_train_original = cPickle.load(fp) \n",
    "\n",
    "    # Test\n",
    "    with open(os.path.join(original_dir, 'X_test_221.pickle'),'rb') as fp: \n",
    "        X_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain complete dataset\n",
    "    X_original = np.concatenate((X_train_original, X_test_original))\n",
    "\n",
    "    # Obtain the images list related to the fold\n",
    "    X_original = X_original[test_indices_list[fold]]\n",
    "\n",
    "    # Resized predictions to the original size\n",
    "    print('Resizing Hybrid predictions to original sizes...')\n",
    "    resized_preds = resize_keypoints_to_original_size(hybrid_pred_kpts, X_original.copy())\n",
    "    print('Hybrid predictions resized.')\n",
    "\n",
    "\n",
    "    # Open y data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'y_train_221.pickle'), 'rb') as fp:\n",
    "        y_train_original = cPickle.load(fp)  \n",
    "\n",
    "    with open(os.path.join(original_dir, 'y_test_221.pickle'), 'rb') as fp:  \n",
    "        y_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain the complete dataset     \n",
    "    y_original = np.concatenate((y_train_original, y_test_original))\n",
    "    \n",
    "    # Obtain the keypoints related to the fold\n",
    "    y_original = y_original[test_indices_list[fold]]\n",
    "\n",
    "\n",
    "    # Create a list append the processing stuff\n",
    "    print('Creating a list to process...')\n",
    "    predictions = [] \n",
    "\n",
    "    # Go through all the images in this fold\n",
    "    for i in range(np.shape(X_original)[0]):\n",
    "        predictions.append(to_strange_fmt(resized_preds[i]))\n",
    "\n",
    "\n",
    "    # Create a scores list to process\n",
    "    print('Creating a scores list to process...')\n",
    "    scores = []\n",
    "\n",
    "    # Go through all the images in this fold\n",
    "    for i in range(np.shape(X_original)[0]): \n",
    "        scores.append(scoring(predictions=predictions[i], y=y_original[i], img_shape=X_original[i].shape, dataset_diagonal=2701.6085, dataset=\"221\"))\n",
    "    \n",
    "    # Generating dense scores\n",
    "    print('Generating final dense scores...')\n",
    "    dense_scores = dense_scoring(scores)\n",
    "\n",
    "    # Print scores\n",
    "    print('MODEL: {} | FOLD: {}'.format('Hybrid', fold))\n",
    "    print('ENDPOINTS [MEAN, STD, MAX]: {}'.format(dense_scores[0]))\n",
    "    print('BREAST CONTOUR [MEAN, STD, MAX]: {}'.format(dense_scores[1]))\n",
    "    print('NIPPLES [MEAN, STD, MAX]: {}\\n'.format(dense_scores[2]))\n",
    "\n",
    "# Finish statement\n",
    "print(\"Hybrid Model Scoring Results finished.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "\n",
    "# Score Utilities Imports\n",
    "from code.scoring.utilities import scoring, dense_scoring, to_strange_fmt, resize_keypoints_to_original_size\n",
    "\n",
    "\n",
    "# GLOBAL VARIABLES\n",
    "FOLDS = [i for i in range(5)]\n",
    "\n",
    "# Directories\n",
    "# Data\n",
    "data_dir = 'data'\n",
    "\n",
    "# Resized Data\n",
    "resized_dir = os.path.join(data_dir, 'resized')\n",
    "\n",
    "# Original Data\n",
    "original_dir = os.path.join(data_dir, 'original')\n",
    "\n",
    "# Train and Test Indices\n",
    "test_indices_list_path = os.path.join(data_dir, 'train-test-indices', 'test_indices_list.pickle')\n",
    "\n",
    "# ISBI Results Directories\n",
    "isbi_results_dir = os.path.join('results', 'isbi-model', 'predictions')\n",
    "\n",
    "# Hybrid Results Directories\n",
    "hybrid_results_dir = os.path.join('results', 'hybrid-model', 'predictions', 'reshaped')\n",
    "\n",
    "\n",
    "# Go trough folds\n",
    "for fold in FOLDS:\n",
    "    print('Current fold {}'.format(fold))\n",
    "\n",
    "    # Open test indices list\n",
    "    with open(test_indices_list_path, 'rb') as t:\n",
    "        test_indices_list = cPickle.load(t)\n",
    "\n",
    "    # Open predictions files\n",
    "    # ISBI Predictions\n",
    "    with open(os.path.join(isbi_results_dir, 'isbi_preds_w_only_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        isbi_pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Keypoints were normalized by the image width (512), so we need to \"denormalize\" them\n",
    "    isbi_pred_kpts = np.array(isbi_pred_kpts)\n",
    "    isbi_pred_kpts *= 512\n",
    "\n",
    "\n",
    "    # Hybrid Predictions\n",
    "    with open(os.path.join(hybrid_results_dir, 'hybrid_preds_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        hybrid_pred_kpts = cPickle.load(f)\n",
    "\n",
    "    # print(hybrid_pred_kpts)\n",
    "\n",
    "    # Convert to NumPy array\n",
    "    hybrid_pred_kpts = np.array(hybrid_pred_kpts)\n",
    "    # print(hybrid_predictions.shape)\n",
    "    \n",
    "    # We have to be sure that the Hybrid Predictions of Endpoints and Nipples are the same as ISBI's\n",
    "    # Endpoints\n",
    "    # truth_lp = y[0:2]\n",
    "    # truth_midl = y[32:34]\n",
    "    # truth_midr = y[66:68]\n",
    "    # truth_rp = y[34:36]\n",
    "    \n",
    "    # Nipples\n",
    "    # truth_l_nipple = y[70:72]\n",
    "    # truth_r_nipple = y[72:74]\n",
    "    \n",
    "    # Begin Process Statement\n",
    "    print('Updating Hybrid Model Endpoints for scoring purposes...')\n",
    "    \n",
    "    # Iterate through ISBI predictions and update Hybrid Predictions Accordingly\n",
    "    for index, value in enumerate(isbi_pred_kpts):\n",
    "        # Endpoints\n",
    "        hybrid_pred_kpts[index][0:2] = value.copy()[0:2]\n",
    "        # hybrid_predictions[index][0:2] = np.round(hybrid_predictions[index][0:2], 5)\n",
    "        # hybrid_predictions[index][1] = value[1]\n",
    "        \n",
    "        hybrid_pred_kpts[index][32:34] = value.copy()[32:34]\n",
    "        # hybrid_predictions[index][32:34] = np.round(hybrid_predictions[index][32:34], 5)\n",
    "        # hybrid_predictions[index][33] = value[33]\n",
    "        \n",
    "        # Mid-points are inversed in hybrid predictions\n",
    "        hybrid_pred_kpts[index][66:68] = value.copy()[66:68]\n",
    "        # hybrid_predictions[index][66:68] = np.round(hybrid_predictions[index][66:68], 5)\n",
    "        # hybrid_predictions[index][67] = value[35]\n",
    "        \n",
    "        hybrid_pred_kpts[index][34:36] = value.copy()[34:36]\n",
    "        # hybrid_predictions[index][34:36] = np.round(hybrid_predictions[index][34:36], 5)\n",
    "        # hybrid_predictions[index][35] = value[67]\n",
    "        \n",
    "        # Nipples\n",
    "        hybrid_pred_kpts[index][70:72] = value.copy()[70:72]\n",
    "        # hybrid_predictions[index][71] = value[71]\n",
    "        hybrid_pred_kpts[index][72:74] = value.copy()[72:74]\n",
    "        # hybrid_predictions[index][73] = value[73] \n",
    "    \n",
    "    print('Hybrid Model Endpoints updated.')\n",
    "\n",
    "\n",
    "    # Open X data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'X_train_221.pickle'), 'rb') as fp: \n",
    "        X_train_original = cPickle.load(fp) \n",
    "\n",
    "    # Test\n",
    "    with open(os.path.join(original_dir, 'X_test_221.pickle'),'rb') as fp: \n",
    "        X_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain complete dataset\n",
    "    X_original = np.concatenate((X_train_original, X_test_original))\n",
    "\n",
    "    # Obtain the images list related to the fold\n",
    "    X_original = X_original[test_indices_list[fold]]\n",
    "\n",
    "    # Resized predictions to the original size\n",
    "    print('Resizing Hybrid predictions to original sizes...')\n",
    "    resized_preds = resize_keypoints_to_original_size(hybrid_pred_kpts, X_original.copy())\n",
    "    print('Hybrid predictions resized.')\n",
    "\n",
    "\n",
    "    # Open y data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'y_train_221.pickle'), 'rb') as fp:\n",
    "        y_train_original = cPickle.load(fp)  \n",
    "\n",
    "    with open(os.path.join(original_dir, 'y_test_221.pickle'), 'rb') as fp:  \n",
    "        y_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain the complete dataset     \n",
    "    y_original = np.concatenate((y_train_original, y_test_original))\n",
    "    \n",
    "    # Obtain the keypoints related to the fold\n",
    "    y_original = y_original[test_indices_list[fold]]\n",
    "    \n",
    "    # Generate Plots\n",
    "    for index, image in enumerate(X_original):\n",
    "        plt.title('ISBI Plots | Fold {} of {} | Image {} of {}'.format(fold, len(FOLDS)-1, index+1, len(images_list[test_indices_list[fold]])))\n",
    "        plt.imshow(image, cmap='gray')\n",
    "        plt.plot(y_original[index][0:74:2], y_original[index][1:75:2], 'bo')\n",
    "        plt.plot(resized_preds[index][0:74:2], resized_preds[index][1:75:2], 'ro')\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Segmentation Based Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scoring Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "\n",
    "# Score Utilities Imports\n",
    "from code.scoring.utilities import scoring, dense_scoring, to_strange_fmt, resize_keypoints_to_original_size\n",
    "\n",
    "\n",
    "# GLOBAL VARIABLES\n",
    "FOLDS = [i for i in range(5)]\n",
    "\n",
    "# Directories\n",
    "# Data\n",
    "data_dir = 'data'\n",
    "\n",
    "# Resized Data\n",
    "resized_dir = os.path.join(data_dir, 'resized')\n",
    "\n",
    "# Original Data\n",
    "original_dir = os.path.join(data_dir, 'original')\n",
    "\n",
    "# Train and Test Indices\n",
    "test_indices_list_path = os.path.join(data_dir, 'train-test-indices', 'test_indices_list.pickle')\n",
    "\n",
    "# ISBI Results Directories\n",
    "isbi_results_dir = os.path.join('results', 'isbi-model', 'predictions')\n",
    "\n",
    "# Segmentation Based Model Results Directories\n",
    "segmentation_based_results_dir = os.path.join('results', 'segmentation-based-model', 'predictions')\n",
    "\n",
    "\n",
    "# Go trough folds\n",
    "for fold in FOLDS:\n",
    "    print('Current fold {}'.format(fold))\n",
    "\n",
    "    # Open test indices list\n",
    "    with open(test_indices_list_path, 'rb') as t:\n",
    "        test_indices_list = cPickle.load(t)\n",
    "\n",
    "    # Open ISBI predictions files\n",
    "    with open(os.path.join(isbi_results_dir, 'isbi_preds_w_only_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        isbi_pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Keypoints were normalized by the image width (512), so we need to \"denormalize\" them\n",
    "    isbi_pred_kpts = np.array(isbi_pred_kpts)\n",
    "    isbi_pred_kpts *= 512\n",
    "\n",
    "\n",
    "    # Open Segmentation Based Model predictions files\n",
    "    with open(os.path.join(segmentation_based_results_dir, 'mixed_preds_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        seg_based_pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Add ISBI Nipples to Segmentation-Based Model Predictions for Scoring Purposes\n",
    "    for index, value in enumerate(isbi_pred_kpts):\n",
    "        # Nipples\n",
    "        seg_based_pred_kpts[index][70] = value[70]\n",
    "        seg_based_pred_kpts[index][71] = value[71]\n",
    "        seg_based_pred_kpts[index][72] = value[72]\n",
    "        seg_based_pred_kpts[index][73] = value[73]\n",
    "\n",
    "\n",
    "    # print(seg_based_pred_kpts)\n",
    "\n",
    "    # Open X data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'X_train_221.pickle'), 'rb') as fp: \n",
    "        X_train_original = cPickle.load(fp) \n",
    "\n",
    "    # Test\n",
    "    with open(os.path.join(original_dir, 'X_test_221.pickle'),'rb') as fp: \n",
    "        X_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain complete dataset\n",
    "    X_original = np.concatenate((X_train_original, X_test_original))\n",
    "\n",
    "    # Obtain the images list related to the fold\n",
    "    X_original = X_original[test_indices_list[fold]]\n",
    "\n",
    "    # Resized predictions to the original size\n",
    "    print('Resizing Segmentation Based Model predictions to original sizes...')\n",
    "    resized_preds = resize_keypoints_to_original_size(seg_based_pred_kpts, X_original.copy())\n",
    "    print('Segmentation Based Model predictions resized.')\n",
    "\n",
    "\n",
    "    # Open y data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'y_train_221.pickle'), 'rb') as fp:\n",
    "        y_train_original = cPickle.load(fp)  \n",
    "\n",
    "    with open(os.path.join(original_dir, 'y_test_221.pickle'), 'rb') as fp:  \n",
    "        y_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain the complete dataset     \n",
    "    y_original = np.concatenate((y_train_original, y_test_original))\n",
    "    \n",
    "    # Obtain the keypoints related to the fold\n",
    "    y_original = y_original[test_indices_list[fold]]\n",
    "\n",
    "\n",
    "    # Create a list append the processing stuff\n",
    "    print('Creating a list to process...')\n",
    "    predictions = [] \n",
    "\n",
    "    # Go through all the images in this fold\n",
    "    for i in range(np.shape(X_original)[0]):\n",
    "        predictions.append(to_strange_fmt(resized_preds[i]))\n",
    "\n",
    "\n",
    "    # Create a scores list to process\n",
    "    print('Creating a scores list to process...')\n",
    "    scores = []\n",
    "\n",
    "    # Go through all the images in this fold\n",
    "    for i in range(np.shape(X_original)[0]): \n",
    "        scores.append(scoring(predictions=predictions[i], y=y_original[i], img_shape=X_original[i].shape, dataset_diagonal=2701.6085, dataset=\"221\"))\n",
    "    \n",
    "    # Generating dense scores\n",
    "    print('Generating final dense scores...')\n",
    "    dense_scores = dense_scoring(scores)\n",
    "\n",
    "    # Print scores\n",
    "    print('MODEL: {} | FOLD: {}'.format('Segmentation Based Model', fold))\n",
    "    print('ENDPOINTS [MEAN, STD, MAX]: {}'.format(dense_scores[0]))\n",
    "    print('BREAST CONTOUR [MEAN, STD, MAX]: {}'.format(dense_scores[1]))\n",
    "    print('NIPPLES [MEAN, STD, MAX]: {}\\n'.format(dense_scores[2]))\n",
    "\n",
    "# Finish statement\n",
    "print(\"Segmentation Based Model Scoring Results finished.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "\n",
    "# Score Utilities Imports\n",
    "from code.scoring.utilities import scoring, dense_scoring, to_strange_fmt, resize_keypoints_to_original_size\n",
    "\n",
    "\n",
    "# GLOBAL VARIABLES\n",
    "FOLDS = [i for i in range(5)]\n",
    "\n",
    "# Directories\n",
    "# Data\n",
    "data_dir = 'data'\n",
    "\n",
    "# Resized Data\n",
    "resized_dir = os.path.join(data_dir, 'resized')\n",
    "\n",
    "# Original Data\n",
    "original_dir = os.path.join(data_dir, 'original')\n",
    "\n",
    "# Train and Test Indices\n",
    "test_indices_list_path = os.path.join(data_dir, 'train-test-indices', 'test_indices_list.pickle')\n",
    "\n",
    "# ISBI Results Directories\n",
    "isbi_results_dir = os.path.join('results', 'isbi-model', 'predictions')\n",
    "\n",
    "# Segmentation Based Model Results Directories\n",
    "segmentation_based_results_dir = os.path.join('results', 'segmentation-based-model', 'predictions')\n",
    "\n",
    "# Segmentation Based Model U-Net++ Predicted Masks Directory\n",
    "segmentation_based_unet_pp_masks_dir = os.path.join('results', 'segmentation-based-model', 'unet-pp', 'predictions')\n",
    "\n",
    "\n",
    "# Go trough folds\n",
    "for fold in FOLDS:\n",
    "    print('Current fold {}'.format(fold))\n",
    "\n",
    "    # Open test indices list\n",
    "    with open(test_indices_list_path, 'rb') as t:\n",
    "        test_indices_list = cPickle.load(t)\n",
    "\n",
    "    # Open ISBI predictions files\n",
    "    with open(os.path.join(isbi_results_dir, 'isbi_preds_w_only_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        isbi_pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Keypoints were normalized by the image width (512), so we need to \"denormalize\" them\n",
    "    isbi_pred_kpts = np.array(isbi_pred_kpts)\n",
    "    isbi_pred_kpts *= 512\n",
    "\n",
    "\n",
    "    # Open Segmentation Based Model predictions files\n",
    "    with open(os.path.join(segmentation_based_results_dir, 'mixed_preds_CV_Fold_{}.pickle'.format(fold)), 'rb') as f:\n",
    "        seg_based_pred_kpts = cPickle.load(f)\n",
    "    \n",
    "    # Add ISBI Nipples to Segmentation-Based Model Predictions for Scoring Purposes\n",
    "    for index, value in enumerate(isbi_pred_kpts):\n",
    "        # Nipples\n",
    "        seg_based_pred_kpts[index][70] = value[70]\n",
    "        seg_based_pred_kpts[index][71] = value[71]\n",
    "        seg_based_pred_kpts[index][72] = value[72]\n",
    "        seg_based_pred_kpts[index][73] = value[73]\n",
    "\n",
    "\n",
    "    # print(seg_based_pred_kpts)\n",
    "\n",
    "    # Open X data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'X_train_221.pickle'), 'rb') as fp: \n",
    "        X_train_original = cPickle.load(fp) \n",
    "\n",
    "    # Test\n",
    "    with open(os.path.join(original_dir, 'X_test_221.pickle'),'rb') as fp: \n",
    "        X_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain complete dataset\n",
    "    X_original = np.concatenate((X_train_original, X_test_original))\n",
    "\n",
    "    # Obtain the images list related to the fold\n",
    "    X_original = X_original[test_indices_list[fold]]\n",
    "\n",
    "    # Resized predictions to the original size\n",
    "    print('Resizing Segmentation Based Model predictions to original sizes...')\n",
    "    resized_preds = resize_keypoints_to_original_size(seg_based_pred_kpts, X_original.copy())\n",
    "    print('Segmentation Based Model predictions resized.')\n",
    "\n",
    "\n",
    "    # Open y data\n",
    "    # Train\n",
    "    with open(os.path.join(original_dir, 'y_train_221.pickle'), 'rb') as fp:\n",
    "        y_train_original = cPickle.load(fp)  \n",
    "\n",
    "    with open(os.path.join(original_dir, 'y_test_221.pickle'), 'rb') as fp:  \n",
    "        y_test_original = cPickle.load(fp)  \n",
    "\n",
    "    # Concatenate both to obtain the complete dataset     \n",
    "    y_original = np.concatenate((y_train_original, y_test_original))\n",
    "    \n",
    "    # Obtain the keypoints related to the fold\n",
    "    y_original = y_original[test_indices_list[fold]]\n",
    "    \n",
    "    \n",
    "    # Open U-Net++ predicted mask to compare keypoints vs masks\n",
    "    unet_pp_masks = os.path.join(segmentation_based_unet_pp_masks_dir, 'unet_pp_preds_CV5_Fold_{}.pickle'.format(fold))\n",
    "    unet_pp_masks = np.reshape(unet_pp_masks, (-1, 384, 512))\n",
    "    unet_pp_masks *= 255\n",
    "    \n",
    "    \n",
    "    \n",
    "    # Generate Plots\n",
    "    print('Segmentation Based Model Plots | Fold {} of {}'.format(fold, len(FOLDS)-1))\n",
    "    for index, image in enumerate(X_original):\n",
    "        \n",
    "        fig, a =  plt.subplots(1,2)\n",
    "        \n",
    "        \n",
    "        # Images Plot\n",
    "        a[0].imshow(image, cmap='gray')\n",
    "        a[0].plot(y_original[index][0:74:2], y_original[index][1:74:2], 'bo')\n",
    "        a[0].plot(resized_preds[index][0:74:2], resized_preds[index][1:75:2], 'ro')\n",
    "        a[0].set_title('Image {} of {}'.format(index+1, len(images_list[test_indices_list[fold]])))\n",
    "        \n",
    "        # Masks Plot\n",
    "        a[1].imshow(unet_pp_masks[index], cmap='gray')\n",
    "        # a[1].plot(gt_keypoints[0:74:2], gt_keypoints[1:74:2], 'bo')\n",
    "        # a[1].plot(resized_segbasedmodel_prediction[0:74:2], resized_segbasedmodel_prediction[1:75:2], 'ro')\n",
    "        a[1].set_title('Mask {} of {}'.format(index+1, len(images_list[test_indices_list[fold]])))\n",
    "        \n",
    "        plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}