nb_analysis.binary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp analysis.binary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Binary analysis\n",
    "\n",
    "> Various methods to modify and work with python dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "from nbdev.showdoc import *\n",
    "from fastcore.test import *\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score, balanced_accuracy_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export \n",
    "def threshold_argmax(y_score_1c, thresh:float=0.5):\n",
    "    '''Converts a sequence of floats into a list of zeros and ones.\n",
    "    \n",
    "    Float values in the sequence are converted to 0 if they are smaller or equal to a set treshold \n",
    "    or 1 if they are larger than the treshold.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    y_score_1c : 1d array-like of floats\n",
    "                 Target scores, can either be probability estimates of the positive class, confidence values, or non-thresholded measure of \n",
    "                 decisions (as returned by “decision_function” on some classifiers). For binary y_true, y_score is supposed to be the score of the class with greater label.\n",
    "    \n",
    "    tresh : float, default=0.5 \n",
    "            Treshold used for converting numbers in sequence\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    y_pred : 1d array-like of integers\n",
    "             Converted scores to integers between 0 and 1\n",
    "    \n",
    "    '''\n",
    "    y_pred = np.array([1 if score > thresh else 0 for score in y_score_1c])\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 1, 1, 1])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold_argmax([-0.1, 0., 0.49, 0.5, 0.51, 1., 1.1 ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export \n",
    "def youdens_jstats(sens:float, spec:float):\n",
    "    '''Calculates Youden's J-statistic \n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    sens : float \n",
    "           Sensitivity \n",
    "    \n",
    "    spec : float\n",
    "           Specificity\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    youden : float\n",
    "             Youden's J-statistic\n",
    "    '''\n",
    "    \n",
    "    youden = sens + spec - 1\n",
    "    return youden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export \n",
    "def performance(y_true, y_score_1c, thresh=0.5, labels:list=[0, 1],\n",
    "                bal_acc:bool=False, youden:bool=False, auroc:bool=False, err_rate:bool=False, bal_err_rate:bool=False):\n",
    "    '''Compute various performance metrics for binary classification predictions\n",
    "    \n",
    "    Computes sensitivity, specificity and accuracy (optional: balanced accuracy, youden's j-statistic, roc auc) given a sequence of target scores\n",
    "    (i.e. probabilities) and the respective ground truth for a binary classification. All metrics are evaluated at a given treshold. \n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    y_true : 1d array-like\n",
    "             Ground truth (correct) target values\n",
    "    \n",
    "    y_score_1c : 1d array-like of floats\n",
    "                 Target scores, can either be probability estimates of the positive class, confidence values, or non-thresholded measure of \n",
    "                 decisions (as returned by “decision_function” on some classifiers). For binary y_true, y_score is supposed to be the score of the class with greater label.\n",
    "             \n",
    "    tresh : float\n",
    "            Treshold\n",
    "            \n",
    "    bal_acc : bool, default=False\n",
    "              Whether to compute or not compute balanced accuracy\n",
    "              \n",
    "    youden : bool, default=False\n",
    "             Whether to compute or not compute Youden's J-statistic\n",
    "              \n",
    "    auroc : bool, default=False\n",
    "            Whether to compute or not compute area under the receiver operating characteristic curve (ROC AUC)   \n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    perf_metrics : dict\n",
    "                   Contains all calculated metrics\n",
    "    '''\n",
    "    y_pred = threshold_argmax(y_score_1c, thresh=thresh)\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=labels).ravel()\n",
    "\n",
    "    # calculate all metrics\n",
    "    perf_metrics = dict()\n",
    "\n",
    "    perf_metrics[\"acc\"] = (tn+tp) / (tn+tp+fn+fp)\n",
    "    perf_metrics[\"sens\"] = tp / (tp+fn)\n",
    "    perf_metrics[\"spec\"] = tn / (tn+fp)\n",
    "    \n",
    "    if bal_acc:\n",
    "        perf_metrics[\"bal_acc\"] = balanced_accuracy_score(y_true, y_pred)\n",
    "    if youden:\n",
    "        perf_metrics[\"youden\"] = youdens_jstats(perf_metrics[\"sens\"], perf_metrics[\"spec\"])\n",
    "    if auroc:\n",
    "        if len(set(y_true.tolist())) > 1:\n",
    "            perf_metrics[\"auroc\"] = roc_auc_score(y_true, y_score_1c)\n",
    "        else:\n",
    "            perf_metrics[\"auroc\"] = np.nan\n",
    "    if err_rate:\n",
    "        perf_metrics[\"err_rate\"] = 1 - perf_metrics[\"acc\"]\n",
    "    if bal_err_rate:\n",
    "        perf_metrics[\"bal_err_rate\"] = 1 - balanced_accuracy_score(y_true, y_pred)\n",
    "    \n",
    "    return perf_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = [0, 1, 0, 1, 0, 1, 0]\n",
    "y_score_1c = [ 0., 0.3, 0.49, 0.5, 0.51, 0.8, 1.]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def auroc(preds, **kwargs):\n",
    "    '''AUROC rate for fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], auroc=True, **kwargs)\n",
    "    return perf[\"auroc\"]\n",
    "\n",
    "def specificity(preds, **kwargs):\n",
    "    '''Specificity for fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], **kwargs)\n",
    "    return perf[\"sens\"]\n",
    "\n",
    "def sensitivity(preds, **kwargs):\n",
    "    '''Sensitivity for fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], **kwargs)\n",
    "    return perf[\"spec\"]\n",
    "\n",
    "def accuracy(preds, **kwargs):\n",
    "    '''Accuracy for fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], **kwargs)\n",
    "    return perf[\"acc\"]\n",
    "\n",
    "def bal_accuracy(preds, **kwargs):\n",
    "    '''Balanced accuracy for  fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], bal_acc=True, **kwargs)\n",
    "    return perf[\"bal_acc\"]\n",
    "\n",
    "def error_rate(preds, **kwargs):\n",
    "    '''Error rate for fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], err_rate=True, **kwargs)\n",
    "    return perf[\"err_rate\"]\n",
    "\n",
    "def bal_error_rate(preds, **kwargs):\n",
    "    '''Balanced error rate for fastai-style-like prediction i.e. tuple of two tensors like (probs, gt)'''\n",
    "    perf = performance(preds[1], preds[0][:,1], bal_err_rate=True, **kwargs)\n",
    "    return perf[\"bal_err_rate\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}