Add features to log precision and recall metric for each class. (detection task)

config/data/local/debug.yaml

@@ -0,0 +1,20 @@
+data:
+  name: debug
+  task: detection
+  format: local # local, huggingface
+  path:
+    root: /workspace/db/data_364_544x960 # dataset root
+    train:
+      image: image/val # directory for training images
+      label: label/val # directory for training labels
+    valid:
+      image: image/val  # directory for valid images
+      label: label/val  # directory for valid labels
+    test:
+      image: image/val #images/val
+      label: label/val #labels/val  # directory for valid labels
+    pattern:
+      image: ~
+      label: ~
+  id_mapping: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13']  # class names
+  pallete: ~

src/netspresso_trainer/metrics/builder.py

@@ -17,15 +17,15 @@
 from typing import Any, Dict
 
 from .base import MetricFactory
-from .registry import METRIC_ADAPTORS, METRIC_LIST, PHASE_LIST, TASK_AVAILABLE_METRICS, TASK_DEFUALT_METRICS
+from .registry import METRIC_ADAPTORS, METRIC_LIST, PHASE_LIST, TASK_AVAILABLE_METRICS, TASK_DEFAULT_METRICS
 
 
 def build_metrics(task: str, model_conf, metrics_conf, num_classes, **kwargs) -> MetricFactory:
     metric_names = metrics_conf.metric_names
     classwise_analysis = metrics_conf.classwise_analysis
 
     if metric_names is None:
-        metric_names = TASK_DEFUALT_METRICS[task]
+        metric_names = TASK_DEFAULT_METRICS[task]
     metric_names = [m.lower() for m in metric_names]
     assert all(metric in TASK_AVAILABLE_METRICS[task] for metric in metric_names), \
         f"Available metrics for {task} are {TASK_AVAILABLE_METRICS[task]}"

src/netspresso_trainer/metrics/detection/__init__.py

@@ -14,4 +14,4 @@
 #
 # ----------------------------------------------------------------------------
 
-from .metric import DetectionMetricAdaptor, mAP50, mAP50_95, mAP75
+from .metric import DetectionMetricAdaptor, Precision, Recall, mAP50, mAP50_95, mAP75

src/netspresso_trainer/metrics/detection/metric.py

@@ -132,7 +132,7 @@ def average_precisions_per_class(
     prediction_confidence: np.ndarray,
     prediction_class_ids: np.ndarray,
     true_class_ids: np.ndarray,
-    num_classes,
+    num_classes: int,
     eps: float = 1e-16,
 ) -> np.ndarray:
     """
@@ -188,6 +188,72 @@ def average_precisions_per_class(
     return average_precisions
 
 
+def precisions_per_class(
+    matches: np.ndarray,
+    prediction_confidence: np.ndarray,
+    prediction_class_ids: np.ndarray,
+    true_class_ids: np.ndarray,
+    num_classes: int,
+    eps: float = 1e-16,
+) -> np.ndarray:
+    sorted_indices = np.argsort(-prediction_confidence)
+    matches = matches[sorted_indices]
+    prediction_class_ids = prediction_class_ids[sorted_indices]
+
+    unique_classes, class_counts = np.unique(true_class_ids, return_counts=True)
+
+    precisions = np.full((num_classes, 1), np.nan)
+
+    for class_idx, class_id in enumerate(unique_classes):
+        is_class = prediction_class_ids == class_id
+        total_true = class_counts[class_idx]
+        total_predictions = is_class.sum()
+
+        if total_true == 0:
+            continue
+        if total_predictions == 0:
+            precisions[int(class_id), 0] = 0.
+            continue
+
+        false_positives = (1 - matches[is_class]).sum(0)
+        true_positives = matches[is_class].sum(0)
+        precision = true_positives / (true_positives + false_positives + eps)
+        precisions[int(class_id), 0] = precision[0]
+    return precisions
+
+def recall_per_class(
+    matches: np.ndarray,
+    prediction_confidence: np.ndarray,
+    prediction_class_ids: np.ndarray,
+    true_class_ids: np.ndarray,
+    num_classes: int,
+    eps: float = 1e-16,
+) -> np.ndarray:
+    sorted_indices = np.argsort(-prediction_confidence)
+    matches = matches[sorted_indices]
+    prediction_class_ids = prediction_class_ids[sorted_indices]
+
+    unique_classes, class_counts = np.unique(true_class_ids, return_counts=True)
+
+    recalls = np.full((num_classes, 1), np.nan)
+
+    for class_idx, class_id in enumerate(unique_classes):
+        is_class = prediction_class_ids == class_id
+        total_true = class_counts[class_idx]
+        total_predictions = is_class.sum()
+
+        if total_true == 0:
+            continue
+        if total_predictions == 0:
+            recalls[int(class_id), 0] = 0.
+            continue
+
+        true_positives = matches[is_class].sum(0)
+        recall = true_positives / (total_true + eps)
+        recalls[int(class_id), 0] = recall[0]
+
+    return recalls
+
 class DetectionMetricAdaptor:
     '''
         Adapter to process redundant operations for the metrics.
@@ -293,3 +359,43 @@ def calibrate(self, predictions, targets, **kwargs):
             self.metric_meter.update(np.nanmean(average_precisions))
         else:
             self.metric_meter.update(0)
+
+
+class Precision(BaseMetric):
+    def __init__(self, num_classes, classwise_analysis, **kwargs):
+        metric_name = 'Precision'
+        super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis)
+
+    def calibrate(self, predictions, targets, **kwargs):
+        stats = kwargs['stats']
+
+        if stats:
+            concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
+            precisions = precisions_per_class(*concatenated_stats, num_classes=self.num_classes)
+
+            if self.classwise_analysis:
+                for i, classwise_meter in enumerate(self.classwise_metric_meters):
+                    classwise_meter.update(np.nanmean(precisions[i, :]))
+            self.metric_meter.update(np.nanmean(precisions))
+        else:
+            self.metric_meter.update(0)
+
+
+class Recall(BaseMetric):
+    def __init__(self, num_classes, classwise_analysis, **kwargs):
+        metric_name = 'Recall'
+        super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis)
+
+    def calibrate(self, predictions, targets, **kwargs):
+        stats = kwargs['stats']
+
+        if stats:
+            concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
+            recalls = recall_per_class(*concatenated_stats, num_classes=self.num_classes)
+
+            if self.classwise_analysis:
+                for i, classwise_meter in enumerate(self.classwise_metric_meters):
+                    classwise_meter.update(np.nanmean(recalls[i, :]))
+            self.metric_meter.update(np.nanmean(recalls))
+        else:
+            self.metric_meter.update(0)

src/netspresso_trainer/metrics/registry.py

@@ -18,7 +18,7 @@
 
 from .base import BaseMetric
 from .classification import ClassificationMetricAdaptor, Top1Accuracy, Top5Accuracy
-from .detection import DetectionMetricAdaptor, mAP50, mAP50_95, mAP75
+from .detection import DetectionMetricAdaptor, Precision, Recall, mAP50, mAP50_95, mAP75
 from .pose_estimation import PCK, PoseEstimationMetricAdaptor
 from .segmentation import PixelAccuracy, SegmentationMetricAdaptor, mIoU
 
@@ -27,6 +27,8 @@
     'top5_accuracy': Top5Accuracy,
     'miou': mIoU,
     'pixel_accuracy': PixelAccuracy,
+    'precision': Precision,
+    'recall': Recall,
     'map50': mAP50,
     'map75': mAP75,
     'map50_95': mAP50_95,
@@ -45,13 +47,13 @@
 TASK_AVAILABLE_METRICS = {
     'classification': ['top1_accuracy', 'top5_accuracy'],
     'segmentation': ['miou', 'pixel_accuracy'],
-    'detection': ['map50', 'map75', 'map50_95'],
+    'detection': ['precision', 'recall', 'map50', 'map75', 'map50_95'],
     'pose_estimation': ['pck'],
 }
 
-TASK_DEFUALT_METRICS = {
+TASK_DEFAULT_METRICS = {
     'classification': ['top1_accuracy', 'top5_accuracy'],
     'segmentation': ['miou', 'pixel_accuracy'],
-    'detection': ['map50', 'map75', 'map50_95'],
+    'detection': ['map50', 'map75', 'map50_95', 'precision', 'recall'],
     'pose_estimation': ['pck'],
 }