challenge_the_challenge_metric.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71d6f853",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log file path is set /home/wenhao/Jupyter/wenhao/workspace/cinc2021/working_dir/CinC2021-CinC2021.log\n",
      "levels of c_handler and f_handler are set DEBUG\n",
      "converting dtypes of columns `diagnosis` and `diagnosis_scored`...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dc68adc1b8a4dbaa233d23ffbac6a1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/85056 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "# ^^^ pyforest auto-imports - don't write above this line\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import random\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from utils.scoring_metrics import evaluate_scores\n",
    "from utils.scoring_aux_data import df_weights_abbr\n",
    "from data_reader import CINC2021Reader\n",
    "from cfg import BaseCfg\n",
    "\n",
    "dr = CINC2021Reader(BaseCfg.db_dir)\n",
    "\n",
    "classes = df_weights_abbr.columns.tolist()\n",
    "\n",
    "n_records = len(dr.df_stats)\n",
    "n_classes = len(classes)\n",
    "\n",
    "truth = np.zeros((n_records, n_classes))\n",
    "binary_pred = np.zeros((n_records,n_classes))\n",
    "scalar_pred = np.random.uniform(0,1, size=(n_records,n_classes))\n",
    "\n",
    "with tqdm(dr.df_stats.iterrows(), total=n_records) as t:\n",
    "    for i, row in t:\n",
    "        labels = dr.get_labels(row.record, scored_only=True, fmt=\"a\")\n",
    "        for l in labels:\n",
    "            truth[i,classes.index(l)] = 1\n",
    "        num = random.randint(2,6)\n",
    "        for j in random.sample(range(n_classes), num):\n",
    "            binary_pred[i,j] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bd0af04",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "86a98516",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- AUROC and AUPRC...\n",
      "- Accuracy...\n",
      "- F-measure...\n",
      "- F-beta and G-beta measures...\n",
      "- Challenge metric...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "auroc, auprc, accuracy, f_measure, f_beta_measure, g_beta_measure, challenge_metric = \\\n",
    "    evaluate_scores(classes, truth, binary_pred, scalar_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa394892",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auroc = 0.50205272550752\n",
      "auprc = 0.05684844911540631\n",
      "accuracy = 8.22987208427389e-05\n",
      "f_measure = 0.0542148543643923\n",
      "f_beta_measure = 0.0574290023798647\n",
      "g_beta_measure = 0.017234829579101395\n",
      "challenge_metric = 0.007172302631995599\n"
     ]
    }
   ],
   "source": [
    "print(\"auroc = {}\\nauprc = {}\\naccuracy = {}\\nf_measure = {}\\nf_beta_measure = {}\\ng_beta_measure = {}\\nchallenge_metric = {}\".format(auroc, auprc, accuracy, f_measure, f_beta_measure, g_beta_measure, challenge_metric))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7dcedb6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}