diff --git a/panoptica/example_spine_matched_instance.ipynb b/panoptica/example_spine_matched_instance.ipynb index 2f46825..e20a019 100644 --- a/panoptica/example_spine_matched_instance.ipynb +++ b/panoptica/example_spine_matched_instance.ipynb @@ -1,401 +1,405 @@ { - "cells": [ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use Case: Matched Instances Input\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Use Case: Matched Instances Input\n" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: panoptica in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (1.0.0.post2.dev0+2f7d01f)\n", + "Requirement already satisfied: auxiliary in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (0.0.42)\n", + "Requirement already satisfied: rich in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (13.6.0)\n", + "Requirement already satisfied: numpy in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (1.25.2)\n", + "Requirement already satisfied: connected-components-3d<4.0.0,>=3.12.3 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (3.12.3)\n", + "Requirement already satisfied: ruamel.yaml<0.19.0,>=0.18.6 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (0.18.6)\n", + "Requirement already satisfied: scikit-image<0.23.0,>=0.22.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (0.22.0)\n", + "Requirement already satisfied: scipy<2.0.0,>=1.7.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (1.11.2)\n", + "Requirement already satisfied: nibabel>=3.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (5.1.0)\n", + "Requirement already satisfied: path>=16.10.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (17.0.0)\n", + "Requirement already satisfied: pathlib>=1.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (1.0.1)\n", + "Requirement already satisfied: pillow>=10.0.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (10.0.0)\n", + "Requirement already satisfied: tifffile>=2023.8.25 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (2023.8.30)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from rich->-r requirements.txt (line 3)) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from rich->-r requirements.txt (line 3)) (2.17.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich->-r requirements.txt (line 3)) (0.1.2)\n", + "Requirement already satisfied: packaging>=17 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from nibabel>=3.0->auxiliary->-r requirements.txt (line 2)) (23.1)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from ruamel.yaml<0.19.0,>=0.18.6->panoptica->-r requirements.txt (line 1)) (0.2.8)\n", + "Requirement already satisfied: networkx>=2.8 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from scikit-image<0.23.0,>=0.22.0->panoptica->-r requirements.txt (line 1)) (3.1)\n", + "Requirement already satisfied: imageio>=2.27 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from scikit-image<0.23.0,>=0.22.0->panoptica->-r requirements.txt (line 1)) (2.31.3)\n", + "Requirement already satisfied: lazy_loader>=0.3 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from scikit-image<0.23.0,>=0.22.0->panoptica->-r requirements.txt (line 1)) (0.3)\n" + ] + } + ], + "source": [ + "!pip install -r requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from auxiliary.nifti.io import read_nifti\n", + "from rich import print as pprint\n", + "from panoptica import InputType, Panoptica_Evaluator\n", + "from panoptica.metrics import Metric" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To demonstrate we use a reference and predicition of spine a segmentation with matched instances.\n", + "\n", + "\n", + "![matched_figure](figures/matched_instance.png)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Install Dependencies" + "data": { + "text/plain": [ + "(array([ 0, 2, 3, 4, 5, 6, 7, 8, 26, 102, 103, 104, 105,\n", + " 106, 107, 108, 202, 203, 204, 205, 206, 207, 208], dtype=uint8),\n", + " array([ 0, 2, 3, 4, 5, 6, 7, 8, 26, 102, 103, 104, 105,\n", + " 106, 107, 108, 202, 203, 204, 205, 206, 207, 208], dtype=uint8))" ] - }, + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ref_masks = read_nifti(\"./spine_seg/matched_instance/ref.nii.gz\")\n", + "pred_masks = read_nifti(\"./spine_seg/matched_instance/pred.nii.gz\")\n", + "\n", + "# labels are matching\n", + "np.unique(ref_masks), np.unique(pred_masks)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: panoptica in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (1.0.0.post2.dev0+2f7d01f)\n", - "Requirement already satisfied: auxiliary in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (0.0.42)\n", - "Requirement already satisfied: rich in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (13.6.0)\n", - "Requirement already satisfied: numpy in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (1.25.2)\n", - "Requirement already satisfied: connected-components-3d<4.0.0,>=3.12.3 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (3.12.3)\n", - "Requirement already satisfied: ruamel.yaml<0.19.0,>=0.18.6 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (0.18.6)\n", - "Requirement already satisfied: scikit-image<0.23.0,>=0.22.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (0.22.0)\n", - "Requirement already satisfied: scipy<2.0.0,>=1.7.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from panoptica->-r requirements.txt (line 1)) (1.11.2)\n", - "Requirement already satisfied: nibabel>=3.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (5.1.0)\n", - "Requirement already satisfied: path>=16.10.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (17.0.0)\n", - "Requirement already satisfied: pathlib>=1.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (1.0.1)\n", - "Requirement already satisfied: pillow>=10.0.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (10.0.0)\n", - "Requirement already satisfied: tifffile>=2023.8.25 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from auxiliary->-r requirements.txt (line 2)) (2023.8.30)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from rich->-r requirements.txt (line 3)) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from rich->-r requirements.txt (line 3)) (2.17.2)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich->-r requirements.txt (line 3)) (0.1.2)\n", - "Requirement already satisfied: packaging>=17 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from nibabel>=3.0->auxiliary->-r requirements.txt (line 2)) (23.1)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from ruamel.yaml<0.19.0,>=0.18.6->panoptica->-r requirements.txt (line 1)) (0.2.8)\n", - "Requirement already satisfied: networkx>=2.8 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from scikit-image<0.23.0,>=0.22.0->panoptica->-r requirements.txt (line 1)) (3.1)\n", - "Requirement already satisfied: imageio>=2.27 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from scikit-image<0.23.0,>=0.22.0->panoptica->-r requirements.txt (line 1)) (2.31.3)\n", - "Requirement already satisfied: lazy_loader>=0.3 in /opt/anaconda3/envs/seg11panoptdev/lib/python3.11/site-packages (from scikit-image<0.23.0,>=0.22.0->panoptica->-r requirements.txt (line 1)) (0.3)\n" - ] - } + "data": { + "text/html": [ + "
────────────────────────────────────────── Thank you for using panoptica ──────────────────────────────────────────\n",
+       "
\n" ], - "source": [ - "!pip install -r requirements.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from auxiliary.nifti.io import read_nifti\n", - "from rich import print as pprint\n", - "from panoptica import InputType, Panoptica_Evaluator\n", - "from panoptica.metrics import Metric" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load Data" + "text/plain": [ + "\u001b[92m────────────────────────────────────────── \u001b[0mThank you for using \u001b[1mpanoptica\u001b[0m\u001b[92m ──────────────────────────────────────────\u001b[0m\n" ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To demonstrate we use a reference and predicition of spine a segmentation with matched instances.\n", - "\n", - "\n", - "![matched_figure](figures/matched_instance.png)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([ 0, 2, 3, 4, 5, 6, 7, 8, 26, 102, 103, 104, 105,\n", - " 106, 107, 108, 202, 203, 204, 205, 206, 207, 208], dtype=uint8),\n", - " array([ 0, 2, 3, 4, 5, 6, 7, 8, 26, 102, 103, 104, 105,\n", - " 106, 107, 108, 202, 203, 204, 205, 206, 207, 208], dtype=uint8))" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } + "data": { + "text/html": [ + "
                                     Please support our development by citing                                      \n",
+       "
\n" ], - "source": [ - "ref_masks = read_nifti(\"./spine_seg/matched_instance/ref.nii.gz\")\n", - "pred_masks = read_nifti(\"./spine_seg/matched_instance/pred.nii.gz\")\n", - "\n", - "# labels are matching\n", - "np.unique(ref_masks), np.unique(pred_masks)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run Evaluation" + "text/plain": [ + " Please support our development by citing \n" ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
────────────────────────────────────────── Thank you for using panoptica ──────────────────────────────────────────\n",
-              "
\n" - ], - "text/plain": [ - "\u001b[92m────────────────────────────────────────── \u001b[0mThank you for using \u001b[1mpanoptica\u001b[0m\u001b[92m ──────────────────────────────────────────\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
                                     Please support our development by citing                                      \n",
-              "
\n" - ], - "text/plain": [ - " Please support our development by citing \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
                          https://github.com/BrainLesion/panoptica#citation -- Thank you!                          \n",
-              "
\n" - ], - "text/plain": [ - " \u001b[4;94mhttps://github.com/BrainLesion/panoptica#citation\u001b[0m -- Thank you! \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
-              "
\n" - ], - "text/plain": [ - "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n",
-              "
\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Panoptic: Start Evaluation\n", - "-- Got MatchedInstancePair, will evaluate instances\n" - ] - } + "data": { + "text/html": [ + "
                          https://github.com/BrainLesion/panoptica#citation -- Thank you!                          \n",
+       "
\n" ], - "source": [ - "evaluator = Panoptica_Evaluator(\n", - " expected_input=InputType.MATCHED_INSTANCE,\n", - " decision_metric=Metric.IOU,\n", - " decision_threshold=0.5,\n", - ")\n", - "\n", - "result, intermediate_steps_data = evaluator.evaluate(pred_masks, ref_masks)[\"ungrouped\"]" + "text/plain": [ + " \u001b[4;94mhttps://github.com/BrainLesion/panoptica#citation\u001b[0m -- Thank you! \n" ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inspect Results\n", - "The results object allows access to individual metrics and provides helper methods for further processing" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "+++ MATCHING +++\n", - "Number of instances in reference (num_ref_instances): 22\n", - "Number of instances in prediction (num_pred_instances): 22\n", - "True Positives (tp): 22\n", - "False Positives (fp): 0\n", - "False Negatives (fn): 0\n", - "Recognition Quality / F1-Score (rq): 1.0\n", - "\n", - "+++ GLOBAL +++\n", - "Global Binary Dice (global_bin_dsc): 0.9744370224078394\n", - "\n", - "+++ INSTANCE +++\n", - "Segmentation Quality IoU (sq): 0.8328184295330796 +- 0.15186064004517466\n", - "Panoptic Quality IoU (pq): 0.8328184295330796\n", - "Segmentation Quality Dsc (sq_dsc): 0.900292616009954 +- 0.10253566174957332\n", - "Panoptic Quality Dsc (pq_dsc): 0.900292616009954\n", - "Segmentation Quality ASSD (sq_assd): 0.250331887879225 +- 0.07696680402317076\n", - "Segmentation Quality Relative Volume Difference (sq_rvd): 0.0028133049062930553 +- 0.034518928495505724\n", - "\n" - ] - } + "data": { + "text/html": [ + "
───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
+       "
\n" ], - "source": [ - "# print all results\n", - "print(result)" + "text/plain": [ + "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
result.pq=0.8328184295330796\n",
-              "
\n" - ], - "text/plain": [ - "result.\u001b[33mpq\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8328184295330796\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } + "data": { + "text/html": [ + "
\n",
+       "
\n" ], - "source": [ - "# get specific metric, e.g. pq\n", - "pprint(f\"{result.pq=}\")" + "text/plain": [ + "\n" ] + }, + "metadata": {}, + "output_type": "display_data" }, { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
results dict: \n",
-              "{\n",
-              "    'num_ref_instances': 22,\n",
-              "    'num_pred_instances': 22,\n",
-              "    'tp': 22,\n",
-              "    'fp': 0,\n",
-              "    'fn': 0,\n",
-              "    'prec': 1.0,\n",
-              "    'rec': 1.0,\n",
-              "    'rq': 1.0,\n",
-              "    'sq': 0.8328184295330796,\n",
-              "    'sq_std': 0.15186064004517466,\n",
-              "    'pq': 0.8328184295330796,\n",
-              "    'sq_dsc': 0.900292616009954,\n",
-              "    'sq_dsc_std': 0.10253566174957332,\n",
-              "    'pq_dsc': 0.900292616009954,\n",
-              "    'sq_assd': 0.250331887879225,\n",
-              "    'sq_assd_std': 0.07696680402317076,\n",
-              "    'sq_rvd': 0.0028133049062930553,\n",
-              "    'sq_rvd_std': 0.034518928495505724,\n",
-              "    'global_bin_dsc': 0.9744370224078394\n",
-              "}\n",
-              "
\n" - ], - "text/plain": [ - "results dict: \n", - "\u001b[1m{\u001b[0m\n", - " \u001b[32m'num_ref_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", - " \u001b[32m'num_pred_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", - " \u001b[32m'tp'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", - " \u001b[32m'fp'\u001b[0m: \u001b[1;36m0\u001b[0m,\n", - " \u001b[32m'fn'\u001b[0m: \u001b[1;36m0\u001b[0m,\n", - " \u001b[32m'prec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", - " \u001b[32m'rec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", - " \u001b[32m'rq'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", - " \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n", - " \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.15186064004517466\u001b[0m,\n", - " \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n", - " \u001b[32m'sq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n", - " \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.10253566174957332\u001b[0m,\n", - " \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n", - " \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.250331887879225\u001b[0m,\n", - " \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.07696680402317076\u001b[0m,\n", - " \u001b[32m'sq_rvd'\u001b[0m: \u001b[1;36m0.0028133049062930553\u001b[0m,\n", - " \u001b[32m'sq_rvd_std'\u001b[0m: \u001b[1;36m0.034518928495505724\u001b[0m,\n", - " \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9744370224078394\u001b[0m\n", - "\u001b[1m}\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } + "name": "stdout", + "output_type": "stream", + "text": [ + "Panoptic: Start Evaluation\n", + "-- Got MatchedInstancePair, will evaluate instances\n" + ] + } + ], + "source": [ + "evaluator = Panoptica_Evaluator(\n", + " expected_input=InputType.MATCHED_INSTANCE,\n", + " decision_metric=Metric.IOU,\n", + " decision_threshold=0.5,\n", + ")\n", + "\n", + "result, intermediate_steps_data = evaluator.evaluate(pred_masks, ref_masks)[\"ungrouped\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inspect Results\n", + "The results object allows access to individual metrics and provides helper methods for further processing" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "+++ MATCHING +++\n", + "Number of instances in reference (num_ref_instances): 22\n", + "Number of instances in prediction (num_pred_instances): 22\n", + "True Positives (tp): 22\n", + "False Positives (fp): 0\n", + "False Negatives (fn): 0\n", + "Recognition Quality / F1-Score (rq): 1.0\n", + "\n", + "+++ GLOBAL +++\n", + "Global Binary Dice (global_bin_dsc): 0.9744370224078394\n", + "\n", + "+++ INSTANCE +++\n", + "Segmentation Quality IoU (sq): 0.8328184295330796 +- 0.15186064004517466\n", + "Panoptic Quality IoU (pq): 0.8328184295330796\n", + "Segmentation Quality Dsc (sq_dsc): 0.900292616009954 +- 0.10253566174957332\n", + "Panoptic Quality Dsc (pq_dsc): 0.900292616009954\n", + "Segmentation Quality ASSD (sq_assd): 0.250331887879225 +- 0.07696680402317076\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): 0.0028133049062930553 +- 0.034518928495505724\n", + "\n" + ] + } + ], + "source": [ + "# print all results\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
result.pq=0.8328184295330796\n",
+       "
\n" ], - "source": [ - "# get dict for further processing, e.g. for pandas\n", - "pprint(\"results dict: \", result.to_dict())" + "text/plain": [ + "result.\u001b[33mpq\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.8328184295330796\u001b[0m\n" ] - }, + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# get specific metric, e.g. pq\n", + "pprint(f\"{result.pq=}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "key SEMANTIC not in intermediate steps, maybe the step was skipped?\n" - ] - } + "data": { + "text/html": [ + "
results dict: \n",
+       "{\n",
+       "    'num_ref_instances': 22,\n",
+       "    'num_pred_instances': 22,\n",
+       "    'tp': 22,\n",
+       "    'fp': 0,\n",
+       "    'fn': 0,\n",
+       "    'prec': 1.0,\n",
+       "    'rec': 1.0,\n",
+       "    'rq': 1.0,\n",
+       "    'sq': 0.8328184295330796,\n",
+       "    'sq_std': 0.15186064004517466,\n",
+       "    'pq': 0.8328184295330796,\n",
+       "    'sq_dsc': 0.900292616009954,\n",
+       "    'sq_dsc_std': 0.10253566174957332,\n",
+       "    'pq_dsc': 0.900292616009954,\n",
+       "    'sq_assd': 0.250331887879225,\n",
+       "    'sq_assd_std': 0.07696680402317076,\n",
+       "    'sq_rvd': 0.0028133049062930553,\n",
+       "    'sq_rvd_std': 0.034518928495505724,\n",
+       "    'global_bin_dsc': 0.9744370224078394\n",
+       "}\n",
+       "
\n" ], - "source": [ - "# To inspect different phases, just use the returned intermediate_steps_data object\n", - "\n", - "intermediate_steps_data.original_prediction_arr # yields input prediction array\n", - "intermediate_steps_data.original_reference_arr # yields input reference array\n", - "\n", - "intermediate_steps_data.prediction_arr(InputType.MATCHED_INSTANCE) # yields prediction array after instances have been matched\n", - "intermediate_steps_data.reference_arr(InputType.MATCHED_INSTANCE) # yields reference array after instances have been matched\n", - "\n", - "# The other InputType do not work here, as the input was already a matched instance map, therefore the steps from instance approximation and matching have been skipped\n", - "try:\n", - " intermediate_steps_data.reference_arr(InputType.SEMANTIC)\n", - "except AssertionError as e:\n", - " print(e)\n", - " # Error will indicate the problem" + "text/plain": [ + "results dict: \n", + "\u001b[1m{\u001b[0m\n", + " \u001b[32m'num_ref_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", + " \u001b[32m'num_pred_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", + " \u001b[32m'tp'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", + " \u001b[32m'fp'\u001b[0m: \u001b[1;36m0\u001b[0m,\n", + " \u001b[32m'fn'\u001b[0m: \u001b[1;36m0\u001b[0m,\n", + " \u001b[32m'prec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", + " \u001b[32m'rec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", + " \u001b[32m'rq'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", + " \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n", + " \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.15186064004517466\u001b[0m,\n", + " \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n", + " \u001b[32m'sq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n", + " \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.10253566174957332\u001b[0m,\n", + " \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n", + " \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.250331887879225\u001b[0m,\n", + " \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.07696680402317076\u001b[0m,\n", + " \u001b[32m'sq_rvd'\u001b[0m: \u001b[1;36m0.0028133049062930553\u001b[0m,\n", + " \u001b[32m'sq_rvd_std'\u001b[0m: \u001b[1;36m0.034518928495505724\u001b[0m,\n", + " \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9744370224078394\u001b[0m\n", + "\u001b[1m}\u001b[0m\n" ] + }, + "metadata": {}, + "output_type": "display_data" } - ], - "metadata": { - "kernelspec": { - "display_name": "helm", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" + ], + "source": [ + "# get dict for further processing, e.g. for pandas\n", + "pprint(\"results dict: \", result.to_dict())" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "key SEMANTIC not in intermediate steps, maybe the step was skipped?\n" + ] } + ], + "source": [ + "# To inspect different phases, just use the returned intermediate_steps_data object\n", + "\n", + "intermediate_steps_data.original_prediction_arr # yields input prediction array\n", + "intermediate_steps_data.original_reference_arr # yields input reference array\n", + "\n", + "intermediate_steps_data.prediction_arr(\n", + " InputType.MATCHED_INSTANCE\n", + ") # yields prediction array after instances have been matched\n", + "intermediate_steps_data.reference_arr(\n", + " InputType.MATCHED_INSTANCE\n", + ") # yields reference array after instances have been matched\n", + "\n", + "# The other InputType do not work here, as the input was already a matched instance map, therefore the steps from instance approximation and matching have been skipped\n", + "try:\n", + " intermediate_steps_data.reference_arr(InputType.SEMANTIC)\n", + "except AssertionError as e:\n", + " print(e)\n", + " # Error will indicate the problem" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "helm", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 2 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/panoptica/example_spine_matching_algorithm.ipynb b/panoptica/example_spine_matching_algorithm.ipynb index bb1eb2c..b623947 100644 --- a/panoptica/example_spine_matching_algorithm.ipynb +++ b/panoptica/example_spine_matching_algorithm.ipynb @@ -171,7 +171,9 @@ "from panoptica import Metric\n", "\n", "# This will match based on IoU metric, will only match if instance have a IoU of 0.5 or higher and will not allow multiple predictions to be matched to the same reference\n", - "matcher = NaiveThresholdMatching(matching_metric=Metric.IOU, matching_threshold=0.5, allow_many_to_one=False)\n", + "matcher = NaiveThresholdMatching(\n", + " matching_metric=Metric.IOU, matching_threshold=0.5, allow_many_to_one=False\n", + ")\n", "\n", "# Now we have to do our processing object ourselves\n", "from panoptica import UnmatchedInstancePair\n", @@ -211,7 +213,9 @@ ], "source": [ "# This will match based on IoU metric, will only match if instance have a IoU of 0.0 or higher and will not allow multiple predictions to be matched to the same reference\n", - "matcher = NaiveThresholdMatching(matching_metric=Metric.IOU, matching_threshold=0.0, allow_many_to_one=False)\n", + "matcher = NaiveThresholdMatching(\n", + " matching_metric=Metric.IOU, matching_threshold=0.0, allow_many_to_one=False\n", + ")\n", "\n", "matched_instance_output = matcher.match_instances(unmatched_instance_input)\n", "print(\"prediction_arr=\", np.unique(matched_instance_output.prediction_arr))\n", diff --git a/panoptica/example_spine_semantic.ipynb b/panoptica/example_spine_semantic.ipynb index dc11110..44bafff 100644 --- a/panoptica/example_spine_semantic.ipynb +++ b/panoptica/example_spine_semantic.ipynb @@ -241,7 +241,9 @@ ], "source": [ "# print all results\n", - "result, intermediate_steps_data = evaluator.evaluate(pred_masks, ref_masks, verbose=False)[\"ungrouped\"]\n", + "result, intermediate_steps_data = evaluator.evaluate(\n", + " pred_masks, ref_masks, verbose=False\n", + ")[\"ungrouped\"]\n", "print(result)" ] }, @@ -380,8 +382,12 @@ "intermediate_steps_data.original_prediction_arr # yields input prediction array\n", "intermediate_steps_data.original_reference_arr # yields input reference array\n", "\n", - "intermediate_steps_data.prediction_arr(InputType.MATCHED_INSTANCE) # yields prediction array after instances have been matched\n", - "intermediate_steps_data.reference_arr(InputType.MATCHED_INSTANCE) # yields reference array after instances have been matched\n", + "intermediate_steps_data.prediction_arr(\n", + " InputType.MATCHED_INSTANCE\n", + ") # yields prediction array after instances have been matched\n", + "intermediate_steps_data.reference_arr(\n", + " InputType.MATCHED_INSTANCE\n", + ") # yields reference array after instances have been matched\n", "\n", "# This works with all InputType\n", "for i in InputType:\n", diff --git a/panoptica/example_spine_unmatched_instance.ipynb b/panoptica/example_spine_unmatched_instance.ipynb index 09af9f5..d994a5a 100644 --- a/panoptica/example_spine_unmatched_instance.ipynb +++ b/panoptica/example_spine_unmatched_instance.ipynb @@ -490,8 +490,15 @@ " print(i)\n", " pred = intermediate_steps_data.prediction_arr(i)\n", " ref = intermediate_steps_data.reference_arr(i)\n", - " print(\"Prediction array shape =\", pred.shape, \"unique_values=\", np.unique(pred))\n", - " print(\"Reference array shape =\", ref.shape, \"unique_values=\", np.unique(ref))\n", + " print(\n", + " \"Prediction array shape =\",\n", + " pred.shape,\n", + " \"unique_values=\",\n", + " np.unique(pred),\n", + " )\n", + " print(\n", + " \"Reference array shape =\", ref.shape, \"unique_values=\", np.unique(ref)\n", + " )\n", " print()\n", " except AssertionError as e:\n", " print(e)\n",