diff --git a/docs/changelog.md b/docs/changelog.md index 2c876755b9..c8a7115535 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -25,6 +25,8 @@ date_modified: 2026-04-23 - Added [#932](https://github.com/roboflow/supervision/pull/932): [`sv.ImageAssets`](https://supervision.roboflow.com/latest/assets/) for downloading sample images alongside existing video assets, useful for examples and tutorials. +- Added [#2247](https://github.com/roboflow/supervision/pull/2247): [`sv.ConfusionMatrix`](https://supervision.roboflow.com/metrics/detection/#confusionmatrix) now accepts `metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES`, computing IoU via `oriented_box_iou_batch` over `xyxyxyxy` coordinates. Previously, OBB inputs silently fell back to axis-aligned bounding-box IoU, producing incorrect match scores for rotated detections. + - Changed [#2169](https://github.com/roboflow/supervision/pull/2169): [`sv.MeanAveragePrecisionResult`](https://supervision.roboflow.com/latest/metrics/mean_average_precision/) and related metric arrays (`mAP_scores`, `ap_per_class`, `iou_thresholds`, precision/recall) are now `float32` instead of `float64`. Reduces memory and speeds up computation; numerical results may differ in the last few digits. - Changed [#2178](https://github.com/roboflow/supervision/pull/2178): [`sv.rle_to_mask`](https://supervision.roboflow.com/latest/detection/utils/converters/#supervision.detection.utils.converters.rle_to_mask) and [`sv.mask_to_rle`](https://supervision.roboflow.com/latest/detection/utils/converters/#supervision.detection.utils.converters.mask_to_rle) moved to `supervision.detection.utils.converters`. The old import path `supervision.dataset.utils` continues to work but is deprecated. diff --git a/src/supervision/metrics/detection.py b/src/supervision/metrics/detection.py index ccebf3e513..6c04613eeb 100644 --- a/src/supervision/metrics/detection.py +++ b/src/supervision/metrics/detection.py @@ -9,30 +9,122 @@ import numpy.typing as npt from deprecate import deprecated_class +from supervision.config import ORIENTED_BOX_COORDINATES from supervision.dataset.core import DetectionDataset from supervision.detection.core import Detections -from supervision.detection.utils.iou_and_nms import box_iou_batch +from supervision.detection.utils.iou_and_nms import ( + box_iou_batch, + oriented_box_iou_batch, +) +from supervision.metrics.core import MetricTarget + + +def _assert_supported_target(metric_target: MetricTarget) -> None: + if metric_target == MetricTarget.MASKS: + raise ValueError( + "MetricTarget.MASKS is not currently supported for ConfusionMatrix." + ) def detections_to_tensor( - detections: Detections, with_confidence: bool = False + detections: Detections, + with_confidence: bool = False, + metric_target: MetricTarget = MetricTarget.BOXES, ) -> npt.NDArray[np.float32]: """ - Convert Supervision Detections to numpy tensors for further computation + Convert Supervision Detections to a numpy tensor for metric computation. Args: - detections: Detections/Targets in the format of sv.Detections - with_confidence: Whether to include confidence in the tensor + detections: Detections/Targets in the format of sv.Detections. + with_confidence: Whether to include confidence as the last column. + metric_target: The type of detection data to use. + Supports `MetricTarget.BOXES` and + `MetricTarget.ORIENTED_BOUNDING_BOXES`. Returns: - Detections as numpy tensors as in (xyxy, class_id, confidence) order + Detections as a float32 numpy array. Shape depends on `metric_target` + and `with_confidence`: + + | `metric_target` | `with_confidence` | shape | + |----------------------------------------|-------------------|-----------| + | `MetricTarget.BOXES` | `False` | `(N, 5)` | + | `MetricTarget.BOXES` | `True` | `(N, 6)` | + | `MetricTarget.ORIENTED_BOUNDING_BOXES` | `False` | `(N, 9)` | + | `MetricTarget.ORIENTED_BOUNDING_BOXES` | `True` | `(N, 10)` | + + Column layout: + + - `BOXES`: ``[x_min, y_min, x_max, y_max, class_id [, confidence]]`` + - `ORIENTED_BOUNDING_BOXES`: + ``[x1, y1, x2, y2, x3, y3, x4, y4, class_id [, confidence]]`` + + Raises: + ValueError: If `metric_target` is `MetricTarget.MASKS`. + ValueError: If `detections.class_id` is `None`. + ValueError: If `with_confidence=True` and `detections.confidence` is `None`. + ValueError: If `metric_target` is `MetricTarget.ORIENTED_BOUNDING_BOXES` + and `detections.data` does not contain `ORIENTED_BOX_COORDINATES`, + or if the stored array does not have exactly `N * 8` elements. + + Examples: + ```pycon + >>> import numpy as np + >>> import supervision as sv + >>> from supervision.metrics.core import MetricTarget + >>> from supervision.config import ORIENTED_BOX_COORDINATES + >>> detections = sv.Detections( + ... xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + ... class_id=np.array([0]), + ... confidence=np.array([0.9]), + ... ) + >>> tensor = detections_to_tensor(detections, with_confidence=True) + >>> tensor.shape + (1, 6) + >>> obb_coords = np.array([[0, 0, 10, 0, 10, 10, 0, 10]], dtype=np.float32) + >>> det_obb = sv.Detections( + ... xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + ... class_id=np.array([0]), + ... data={ORIENTED_BOX_COORDINATES: obb_coords}, + ... ) + >>> tensor_obb = detections_to_tensor( + ... det_obb, metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES + ... ) + >>> tensor_obb.shape + (1, 9) + + ``` """ + _assert_supported_target(metric_target) + if detections.class_id is None: raise ValueError( "ConfusionMatrix can only be calculated for Detections with class_id" ) - arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + obb = detections.data.get(ORIENTED_BOX_COORDINATES) + if obb is None: + if len(detections) > 0: + raise ValueError( + "ORIENTED_BOUNDING_BOXES requested, but " + f"{ORIENTED_BOX_COORDINATES} is missing from detections.data" + ) + box_data = np.empty((0, 8), dtype=np.float32) + else: + obb_arr = np.asarray(obb, dtype=np.float32) + if obb_arr.size != len(detections) * 8: + raise ValueError( + f"Expected {ORIENTED_BOX_COORDINATES} to contain " + f"{len(detections) * 8} elements " + f"(N={len(detections)} detections x 8 coordinates), " + f"but got {obb_arr.size}. " + "Each OBB must be stored as [x1, y1, x2, y2, x3, y3, x4, y4]." + ) + box_data = obb_arr.reshape(-1, 8) + else: + box_data = detections.xyxy + + arrays_to_concat = [box_data, np.expand_dims(detections.class_id, 1)] if with_confidence: if detections.confidence is None: @@ -48,10 +140,12 @@ def detections_to_tensor( def validate_input_tensors( predictions: list[npt.NDArray[np.float32]], targets: list[npt.NDArray[np.float32]], + metric_target: MetricTarget = MetricTarget.BOXES, ) -> None: """ Checks for shape consistency of input tensors. """ + _assert_supported_target(metric_target) if len(predictions) != len(targets): raise ValueError( f"Number of predictions ({len(predictions)}) and" @@ -62,17 +156,26 @@ def validate_input_tensors( targets[0], np.ndarray ): raise ValueError( - f"Predictions and targets must be lists of numpy arrays." + "Predictions and targets must be lists of numpy arrays. " f"Got {type(predictions[0])} and {type(targets[0])} instead." ) - if predictions[0].shape[1] != 6: + + expected_pred_cols = ( + 10 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 6 + ) + expected_target_cols = ( + 9 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 5 + ) + + if predictions[0].shape[1] != expected_pred_cols: raise ValueError( - f"Predictions must have shape (N, 6)." + f"Predictions must have shape (N, {expected_pred_cols}). " f"Got {predictions[0].shape} instead." ) - if targets[0].shape[1] != 5: + if targets[0].shape[1] != expected_target_cols: raise ValueError( - f"Targets must have shape (N, 5). Got {targets[0].shape} instead." + f"Targets must have shape (N, {expected_target_cols}). " + f"Got {targets[0].shape} instead." ) @@ -89,12 +192,15 @@ class ConfusionMatrix: Detections with lower confidence will be excluded from the matrix. iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. + metric_target: The type of detection data used for IoU computation. + Informational metadata set by `from_detections` and `from_tensors`. """ matrix: npt.NDArray[np.int32] classes: list[str] conf_threshold: float iou_threshold: float + metric_target: MetricTarget = MetricTarget.BOXES @classmethod def from_detections( @@ -104,6 +210,7 @@ def from_detections( classes: list[str], conf_threshold: float = 0.3, iou_threshold: float = 0.5, + metric_target: MetricTarget = MetricTarget.BOXES, ) -> ConfusionMatrix: """ Calculate confusion matrix based on predicted and ground-truth detections. @@ -116,6 +223,14 @@ def from_detections( Detections with lower confidence will be excluded. iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. + metric_target: The type of detection data to use. + Supports `MetricTarget.BOXES` (default) and + `MetricTarget.ORIENTED_BOUNDING_BOXES`. When using + `MetricTarget.ORIENTED_BOUNDING_BOXES`, each `Detections` + object must include OBB coordinates in + `detections.data[ORIENTED_BOX_COORDINATES]` as a float32 + array of shape `(N, 8)` or `(N, 4, 2)`. + `MetricTarget.MASKS` is not supported. Returns: New instance of ConfusionMatrix. @@ -126,15 +241,15 @@ def from_detections( >>> import supervision as sv >>> targets = [ ... sv.Detections( - ... xyxy=np.array([[0, 0, 10, 10]]), - ... class_id=np.array([0]) + ... xyxy=np.array([[0, 0, 10, 10], [50, 50, 60, 60]]), + ... class_id=np.array([0, 0]) ... ) ... ] >>> predictions = [ ... sv.Detections( - ... xyxy=np.array([[0, 0, 10, 10]]), - ... class_id=np.array([0]), - ... confidence=np.array([0.9]) + ... xyxy=np.array([[0, 0, 10, 10], [100, 100, 110, 110]]), + ... class_id=np.array([0, 0]), + ... confidence=np.array([0.9, 0.8]) ... ) ... ] >>> confusion_matrix = sv.ConfusionMatrix.from_detections( @@ -143,8 +258,8 @@ def from_detections( ... classes=['person'] ... ) >>> confusion_matrix.matrix - array([[1., 0.], - [0., 0.]]) + array([[1., 1.], + [1., 0.]]) ``` """ @@ -152,15 +267,22 @@ def from_detections( target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - detections_to_tensor(prediction, with_confidence=True) + detections_to_tensor( + prediction, with_confidence=True, metric_target=metric_target + ) + ) + target_tensors.append( + detections_to_tensor( + target, with_confidence=False, metric_target=metric_target + ) ) - target_tensors.append(detections_to_tensor(target, with_confidence=False)) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, classes=classes, conf_threshold=conf_threshold, iou_threshold=iou_threshold, + metric_target=metric_target, ) @classmethod @@ -171,24 +293,34 @@ def from_tensors( classes: list[str], conf_threshold: float = 0.3, iou_threshold: float = 0.5, + metric_target: MetricTarget = MetricTarget.BOXES, ) -> ConfusionMatrix: """ Calculate confusion matrix based on predicted and ground-truth detections. Args: predictions: Each element of the list describes a single - image and has `shape = (M, 6)` where `M` is the number of detected - objects. Each row is expected to be in + image and has `shape = (M, 6)` or `shape = (M, 10)` depending on + `metric_target`. + If `MetricTarget.BOXES`, each row is in `(x_min, y_min, x_max, y_max, class, conf)` format. + If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in + `(x1, y1, x2, y2, x3, y3, x4, y4, class, conf)` format. targets: Each element of the list describes a single - image and has `shape = (N, 5)` where `N` is the number of - ground-truth objects. Each row is expected to be in + image and has `shape = (N, 5)` or `shape = (N, 9)` depending on + `metric_target`. + If `MetricTarget.BOXES`, each row is in `(x_min, y_min, x_max, y_max, class)` format. + If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in + `(x1, y1, x2, y2, x3, y3, x4, y4, class)` format. classes: Model class names. conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold: Detection iou threshold between `0` and `1`. + iou_threshold: Detection iou threshold between `0` and `1`. Detections with lower iou will be classified as `FP`. + metric_target: The type of detection data to use. + Determines expected tensor shapes (see Args above for column + layouts). `MetricTarget.MASKS` is not supported. Returns: New instance of ConfusionMatrix. @@ -223,7 +355,7 @@ def from_tensors( ``` """ - validate_input_tensors(predictions, targets) + validate_input_tensors(predictions, targets, metric_target=metric_target) num_classes = len(classes) matrix = np.zeros((num_classes + 1, num_classes + 1)) @@ -234,12 +366,14 @@ def from_tensors( num_classes=num_classes, conf_threshold=conf_threshold, iou_threshold=iou_threshold, + metric_target=metric_target, ) return cls( matrix=matrix, classes=classes, conf_threshold=conf_threshold, iou_threshold=iou_threshold, + metric_target=metric_target, ) @staticmethod @@ -249,38 +383,52 @@ def evaluate_detection_batch( num_classes: int, conf_threshold: float, iou_threshold: float, + metric_target: MetricTarget = MetricTarget.BOXES, ) -> npt.NDArray[np.int32]: """ Calculate confusion matrix for a batch of detections for a single image. Args: predictions: Batch prediction. Describes a single image and - has `shape = (M, 6)` where `M` is the number of detected objects. - Each row is expected to be in + has `shape = (M, 6)` or `shape = (M, 10)` depending on + `metric_target`. + If `MetricTarget.BOXES`, each row is in `(x_min, y_min, x_max, y_max, class, conf)` format. + If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in + `(x1, y1, x2, y2, x3, y3, x4, y4, class, conf)` format. targets: Batch target labels. Describes a single image and - has `shape = (N, 5)` where `N` is the number of ground-truth objects. - Each row is expected to be in + has `shape = (N, 5)` or `shape = (N, 9)` depending on + `metric_target`. + If `MetricTarget.BOXES`, each row is in `(x_min, y_min, x_max, y_max, class)` format. + If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in + `(x1, y1, x2, y2, x3, y3, x4, y4, class)` format. num_classes: Number of classes. conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold: Detection iou threshold between `0` and `1`. + iou_threshold: Detection iou threshold between `0` and `1`. Detections with lower iou will be classified as `FP`. + metric_target: The type of detection data to use. + Determines IoU function (`box_iou_batch` vs + `oriented_box_iou_batch`) and coordinate column count. + `MetricTarget.MASKS` is not supported. Returns: Confusion matrix based on a single image. """ + _assert_supported_target(metric_target) result_matrix = np.zeros((num_classes + 1, num_classes + 1)) # Filter predictions by confidence threshold - conf_idx = 5 + coords_dim = 8 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 4 + class_id_idx = coords_dim + conf_idx = coords_dim + 1 + confidence = predictions[:, conf_idx] detection_batch_filtered = predictions[confidence >= conf_threshold] if len(detection_batch_filtered) == 0: # No detections pass confidence threshold - all GT are FN - class_id_idx = 4 true_classes = np.array(targets[:, class_id_idx], dtype=np.int16) for gt_class in true_classes: result_matrix[gt_class, num_classes] += 1 @@ -288,7 +436,6 @@ def evaluate_detection_batch( if len(targets) == 0: # No ground truth - all detections are FP - class_id_idx = 4 detection_classes = np.array( detection_batch_filtered[:, class_id_idx], dtype=np.int16 ) @@ -296,18 +443,22 @@ def evaluate_detection_batch( result_matrix[num_classes, det_class] += 1 return result_matrix - class_id_idx = 4 true_classes = np.array(targets[:, class_id_idx], dtype=np.int16) detection_classes = np.array( detection_batch_filtered[:, class_id_idx], dtype=np.int16 ) - true_boxes = targets[:, :class_id_idx] - detection_boxes = detection_batch_filtered[:, :class_id_idx] + true_boxes = targets[:, :coords_dim] + detection_boxes = detection_batch_filtered[:, :coords_dim] # Calculate IoU matrix - iou_batch = box_iou_batch( - boxes_true=true_boxes, boxes_detection=detection_boxes - ) + if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + iou_batch = oriented_box_iou_batch( + boxes_true=true_boxes, boxes_detection=detection_boxes + ) + else: + iou_batch = box_iou_batch( + boxes_true=true_boxes, boxes_detection=detection_boxes + ) # Find all valid matches (IoU > threshold, regardless of class) # Use vectorized operations to avoid nested Python loops @@ -393,6 +544,7 @@ def benchmark( callback: Callable[[npt.NDArray[np.uint8]], Detections], conf_threshold: float = 0.3, iou_threshold: float = 0.5, + metric_target: MetricTarget = MetricTarget.BOXES, ) -> ConfusionMatrix: """ Calculate confusion matrix from dataset and callback function. @@ -405,6 +557,10 @@ def benchmark( Detections with lower confidence will be excluded. iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. + metric_target: The type of detection data to use. + Supports `MetricTarget.BOXES` and + `MetricTarget.ORIENTED_BOUNDING_BOXES`. Passed through to + `from_detections`. `MetricTarget.MASKS` is not supported. Returns: New instance of ConfusionMatrix. @@ -446,6 +602,7 @@ def callback(image: np.ndarray) -> sv.Detections: classes=dataset.classes, conf_threshold=conf_threshold, iou_threshold=iou_threshold, + metric_target=metric_target, ) def plot( diff --git a/tests/metrics/test_detection.py b/tests/metrics/test_detection.py index ba7b1f9d6a..48bddaba09 100644 --- a/tests/metrics/test_detection.py +++ b/tests/metrics/test_detection.py @@ -8,10 +8,12 @@ from supervision.dataset.core import DetectionDataset from supervision.detection.core import Detections +from supervision.metrics.core import MetricTarget from supervision.metrics.detection import ( ConfusionMatrix, MeanAveragePrecision, detections_to_tensor, + validate_input_tensors, ) from tests.helpers import ( _create_detections, @@ -20,6 +22,46 @@ ) +def _call_confusion_matrix_from_detections_masks() -> None: + ConfusionMatrix.from_detections( + predictions=[ + Detections( + xyxy=np.zeros((1, 4), dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.9]), + ) + ], + targets=[ + Detections( + xyxy=np.zeros((1, 4), dtype=np.float32), + class_id=np.array([0]), + ) + ], + classes=["box"], + metric_target=MetricTarget.MASKS, + ) + + +def _call_confusion_matrix_from_tensors_masks() -> None: + ConfusionMatrix.from_tensors( + predictions=[np.zeros((1, 6), dtype=np.float32)], + targets=[np.zeros((1, 5), dtype=np.float32)], + classes=["box"], + metric_target=MetricTarget.MASKS, + ) + + +def _call_confusion_matrix_evaluate_detection_batch_masks() -> None: + ConfusionMatrix.evaluate_detection_batch( + predictions=np.zeros((1, 6), dtype=np.float32), + targets=np.zeros((1, 5), dtype=np.float32), + num_classes=1, + conf_threshold=0.3, + iou_threshold=0.5, + metric_target=MetricTarget.MASKS, + ) + + class TestDetectionMetrics: """ Verify that detection metrics are computed accurately. @@ -141,25 +183,48 @@ def worsen_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray | li ) @pytest.mark.parametrize( - ("detections", "with_confidence", "expected_result", "exception"), + ( + "detections", + "with_confidence", + "metric_target", + "expected_result", + "exception", + ), [ ( Detections.empty(), False, + MetricTarget.BOXES, np.empty((0, 5), dtype=np.float32), DoesNotRaise(), ), # empty detections; no confidence ( Detections.empty(), True, + MetricTarget.BOXES, np.empty((0, 6), dtype=np.float32), DoesNotRaise(), ), # empty detections; with confidence + ( + Detections.empty(), + False, + MetricTarget.ORIENTED_BOUNDING_BOXES, + np.empty((0, 9), dtype=np.float32), + DoesNotRaise(), + ), # empty OBB detections; no confidence + ( + Detections.empty(), + True, + MetricTarget.ORIENTED_BOUNDING_BOXES, + np.empty((0, 10), dtype=np.float32), + DoesNotRaise(), + ), # empty OBB detections; with confidence ( _create_detections( xyxy=[[0, 0, 10, 10]], class_id=[0], confidence=[0.5] ), False, + MetricTarget.BOXES, np.array([[0, 0, 10, 10, 0]], dtype=np.float32), DoesNotRaise(), ), # single detection; no confidence @@ -168,9 +233,64 @@ def worsen_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray | li xyxy=[[0, 0, 10, 10]], class_id=[0], confidence=[0.5] ), True, + MetricTarget.BOXES, np.array([[0, 0, 10, 10, 0, 0.5]], dtype=np.float32), DoesNotRaise(), ), # single detection; with confidence + ( + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.5], dtype=np.float32), + data={ + "xyxyxyxy": np.array( + [[[0, 0], [10, 0], [10, 10], [0, 10]]], dtype=np.float32 + ) + }, + ), + False, + MetricTarget.ORIENTED_BOUNDING_BOXES, + np.array([[0, 0, 10, 0, 10, 10, 0, 10, 0]], dtype=np.float32), + DoesNotRaise(), + ), # single OBB detection; no confidence + ( + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.5], dtype=np.float32), + data={ + "xyxyxyxy": np.array( + [[[0, 0], [10, 0], [10, 10], [0, 10]]], dtype=np.float32 + ) + }, + ), + True, + MetricTarget.ORIENTED_BOUNDING_BOXES, + np.array([[0, 0, 10, 0, 10, 10, 0, 10, 0, 0.5]], dtype=np.float32), + DoesNotRaise(), + ), # single OBB detection; with confidence + ( + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.5], dtype=np.float32), + ), + False, + MetricTarget.ORIENTED_BOUNDING_BOXES, + None, + pytest.raises(ValueError, match="ORIENTED_BOUNDING_BOXES requested"), + ), # OBB requested but data missing + ( + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.5], dtype=np.float32), + ), + False, + MetricTarget.MASKS, + None, + pytest.raises(ValueError, match=r"MetricTarget\.MASKS"), + ), # MASKS requested but not supported ( _create_detections( xyxy=[[0, 0, 10, 10], [0, 0, 20, 20]], @@ -178,6 +298,7 @@ def worsen_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray | li confidence=[0.5, 0.2], ), False, + MetricTarget.BOXES, np.array([[0, 0, 10, 10, 0], [0, 0, 20, 20, 1]], dtype=np.float32), DoesNotRaise(), ), # multiple detections; no confidence @@ -188,6 +309,7 @@ def worsen_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray | li confidence=[0.5, 0.2], ), True, + MetricTarget.BOXES, np.array( [[0, 0, 10, 10, 0, 0.5], [0, 0, 20, 20, 1, 0.2]], dtype=np.float32 ), @@ -199,6 +321,7 @@ def test_detections_to_tensor( self, detections: Detections, with_confidence: bool, + metric_target: MetricTarget, expected_result: np.ndarray | None, exception: Exception, ) -> None: @@ -211,9 +334,11 @@ def test_detections_to_tensor( """ with exception: result = detections_to_tensor( - detections=detections, with_confidence=with_confidence + detections=detections, + with_confidence=with_confidence, + metric_target=metric_target, ) - assert np.array_equal(result, expected_result) + np.testing.assert_allclose(result, expected_result, atol=1e-5) @pytest.mark.parametrize( ( @@ -1033,9 +1158,346 @@ class agreement) works correctly when applied to a dataset loaded from total_fp = confusion_matrix.matrix[: len(classes), -1].sum() assert total_fp >= 0, f"FP count negative ({total_fp}), computation bug" - # Verify IoU+class fix: wrong-class preds should become FPs, not match GTs + # Verify IoU+class fix: wrong-class preds should become FPs, + # not match GTs assert total_fp > 0 or total_tp == total_gt, ( f"Expected FPs from wrong-class preds (got {total_fp}) or all GTs " f"matched (TP={total_tp}, GT={total_gt}). IoU+class fix may be broken: " f"wrong-class preds with high IoU might incorrectly match GTs." ) + + @pytest.mark.parametrize( + ("predictions", "targets", "metric_target", "exception"), + [ + ( + [np.zeros((1, 10), dtype=np.float32)], + [np.zeros((1, 9), dtype=np.float32)], + MetricTarget.ORIENTED_BOUNDING_BOXES, + DoesNotRaise(), + ), + ( + [np.zeros((1, 6), dtype=np.float32)], + [np.zeros((1, 5), dtype=np.float32)], + MetricTarget.ORIENTED_BOUNDING_BOXES, + pytest.raises(ValueError, match="Predictions must have shape"), + ), + ( + [np.zeros((1, 10), dtype=np.float32)], + [np.zeros((1, 9), dtype=np.float32)], + MetricTarget.BOXES, + pytest.raises(ValueError, match="Predictions must have shape"), + ), + ], + ) + def test_validate_input_tensors_obb( + self, predictions, targets, metric_target, exception + ): + + with exception: + validate_input_tensors(predictions, targets, metric_target=metric_target) + + def test_confusion_matrix_obb(self): + """ + Verify OBB support in ConfusionMatrix. + Test scenarios: + 1. Perfect OBB overlap (Rotation Match) + 2. Rotation Sensitivity (Same AABB, different rotation) + 3. Regression (BOXES mode) + """ + classes = ["box"] + + # Perfect OBB overlap + # 45 degree rotated box + obb_coords = np.array( + [[[5, 0], [10, 5], [5, 10], [0, 5]]], dtype=np.float32 + ) # Diamond shape + gt = [ + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + data={"xyxyxyxy": obb_coords}, + ) + ] + pred = [ + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.9]), + data={"xyxyxyxy": obb_coords}, + ) + ] + + cm_obb = ConfusionMatrix.from_detections( + predictions=pred, + targets=gt, + classes=classes, + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES, + ) + # Expected TP = 1 + assert cm_obb.matrix[0, 0] == 1 + assert cm_obb.matrix.sum() == 1 + + # Rotation Sensitivity + # GT is 45 deg, Pred is axis-aligned box with same AABB + aabb_coords = np.array([[[0, 0], [10, 0], [10, 10], [0, 10]]], dtype=np.float32) + pred_aabb = [ + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + confidence=np.array([0.9]), + data={"xyxyxyxy": aabb_coords}, + ) + ] + + # In OBB mode, IoU between diamond and square is 0.5 + cm_sensitivity_obb = ConfusionMatrix.from_detections( + predictions=pred_aabb, + targets=gt, + classes=classes, + iou_threshold=0.6, # Threshold higher than 0.5 + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES, + ) + # Expected FN=1, FP=1 (no match because IoU=0.5 < 0.6) + assert cm_sensitivity_obb.matrix[0, 1] == 1 # FN + assert cm_sensitivity_obb.matrix[1, 0] == 1 # FP + + # In BOXES mode, they match perfectly + cm_sensitivity_boxes = ConfusionMatrix.from_detections( + predictions=pred_aabb, + targets=gt, + classes=classes, + iou_threshold=0.6, + metric_target=MetricTarget.BOXES, + ) + # Expected TP = 1 + assert cm_sensitivity_boxes.matrix[0, 0] == 1 + + # Deterministic comparison: OBB IoU should be less than AABB IoU + # Here OBB IoU is 0.5, AABB IoU is 1.0 + # We can verify this by checking that a threshold between them + # differentiates behavior + assert cm_sensitivity_obb.matrix[0, 0] == 0 + assert cm_sensitivity_boxes.matrix[0, 0] == 1 + + def test_confusion_matrix_obb_regression_1760(self): + """Regression for #1760: thin OBBs with same AABB must not match. + + Two thin bars (100 px long, 10 px wide) at 45° and -45° share an identical + AABB (AABB IoU = 1.0), but their actual OBB overlap is only at the crossing + centre (OBB IoU ≈ 0.05). + + With iou_threshold=0.5: + - ORIENTED_BOUNDING_BOXES mode: no match → FP + FN (bug before fix: TP) + - BOXES mode: AABB IoU=1.0 → TP (controls: confirms AABB path unbroken) + + A regression that swaps oriented_box_iou_batch back to box_iou_batch would + flip the OBB assertion to TP, surfacing the exact bug from issue #1760. + """ + classes = ["bar"] + sq2 = np.float32(1.0 / np.sqrt(2)) + cx, cy = np.float32(100.0), np.float32(100.0) + hl, hw = np.float32(50.0), np.float32(5.0) + + # Bar at 45°: length direction (sq2, sq2), width direction (-sq2, sq2) + bar_45 = np.array( + [ + [cx + (hl - hw) * sq2, cy + (hl + hw) * sq2], + [cx + (hl + hw) * sq2, cy + (hl - hw) * sq2], + [cx - (hl - hw) * sq2, cy - (hl + hw) * sq2], + [cx - (hl + hw) * sq2, cy - (hl - hw) * sq2], + ], + dtype=np.float32, + ) + # Bar at -45°: length direction (sq2, -sq2), width direction (sq2, sq2) + bar_neg45 = np.array( + [ + [cx + (hl + hw) * sq2, cy - (hl - hw) * sq2], + [cx + (hl - hw) * sq2, cy - (hl + hw) * sq2], + [cx - (hl + hw) * sq2, cy + (hl - hw) * sq2], + [cx - (hl - hw) * sq2, cy + (hl + hw) * sq2], + ], + dtype=np.float32, + ) + + # Both bars share the same axis-aligned bounding box + half_aabb = (hl + hw) * sq2 + shared_xyxy = np.array( + [[cx - half_aabb, cy - half_aabb, cx + half_aabb, cy + half_aabb]], + dtype=np.float32, + ) + + gt = [ + Detections( + xyxy=shared_xyxy, + class_id=np.array([0]), + data={"xyxyxyxy": bar_45[np.newaxis]}, + ) + ] + pred = [ + Detections( + xyxy=shared_xyxy, + class_id=np.array([0]), + confidence=np.array([0.9], dtype=np.float32), + data={"xyxyxyxy": bar_neg45[np.newaxis]}, + ) + ] + + # OBB mode: orthogonal bars barely overlap → FP and FN, not TP + cm_obb = ConfusionMatrix.from_detections( + predictions=pred, + targets=gt, + classes=classes, + iou_threshold=0.5, + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES, + ) + assert cm_obb.matrix[0, 0] == 0 # no TP + assert cm_obb.matrix[0, 1] == 1 # FN + assert cm_obb.matrix[1, 0] == 1 # FP + + # BOXES mode: identical AABB → TP (controls that AABB path is intact) + cm_boxes = ConfusionMatrix.from_detections( + predictions=pred, + targets=gt, + classes=classes, + iou_threshold=0.5, + metric_target=MetricTarget.BOXES, + ) + assert cm_boxes.matrix[0, 0] == 1 + assert cm_boxes.matrix.sum() == 1 + + @pytest.mark.parametrize( + ("pred_tensor", "target_tensor", "iou_threshold", "expected_matrix"), + [ + pytest.param( + np.array([[5, 0, 10, 5, 5, 10, 0, 5, 0, 0.9]], dtype=np.float32), + np.array([[5, 0, 10, 5, 5, 10, 0, 5, 0]], dtype=np.float32), + 0.5, + np.array([[1.0, 0.0], [0.0, 0.0]]), + id="perfect_match_tp", + ), + pytest.param( + np.array([[0, 0, 10, 0, 10, 10, 0, 10, 0, 0.9]], dtype=np.float32), + np.array([[5, 0, 10, 5, 5, 10, 0, 5, 0]], dtype=np.float32), + 0.3, + np.array([[1.0, 0.0], [0.0, 0.0]]), + id="partial_obb_match_tp_at_threshold_0_3", + ), + pytest.param( + np.array([[0, 0, 10, 0, 10, 10, 0, 10, 0, 0.9]], dtype=np.float32), + np.array([[5, 0, 10, 5, 5, 10, 0, 5, 0]], dtype=np.float32), + 0.7, + np.array([[0.0, 1.0], [1.0, 0.0]]), + id="partial_obb_no_match_at_threshold_0_7", + ), + ], + ) + def test_confusion_matrix_from_tensors_obb( + self, pred_tensor, target_tensor, iou_threshold, expected_matrix + ): + """Direct from_tensors OBB end-to-end coverage with pre-built tensors.""" + cm = ConfusionMatrix.from_tensors( + predictions=[pred_tensor], + targets=[target_tensor], + classes=["box"], + iou_threshold=iou_threshold, + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES, + ) + np.testing.assert_array_equal(cm.matrix, expected_matrix) + + @pytest.mark.parametrize( + "call", + [ + pytest.param( + _call_confusion_matrix_from_detections_masks, + id="from_detections", + ), + pytest.param( + _call_confusion_matrix_from_tensors_masks, + id="from_tensors", + ), + pytest.param( + _call_confusion_matrix_evaluate_detection_batch_masks, + id="evaluate_detection_batch", + ), + ], + ) + def test_confusion_matrix_masks_rejection(self, call): + """MetricTarget.MASKS raises ValueError at every public entry point.""" + with pytest.raises(ValueError, match=r"MetricTarget\.MASKS"): + call() + + def test_confusion_matrix_multiclass_obb(self): + """Multi-class OBB: TP for exact match, FP+FN for rotation mismatch.""" + classes = ["box", "circle"] + + # Ground truth: class-0 diamond at [0,0,10,10], class-1 diamond at [20,20,30,30] + gt = [ + Detections( + xyxy=np.array([[0, 0, 10, 10], [20, 20, 30, 30]], dtype=np.float32), + class_id=np.array([0, 1]), + data={ + "xyxyxyxy": np.array( + [ + [[5, 0], [10, 5], [5, 10], [0, 5]], + [[25, 20], [30, 25], [25, 30], [20, 25]], + ], + dtype=np.float32, + ) + }, + ) + ] + # Predictions: class-0 same diamond (OBB IoU=1.0 → TP); + # class-1 axis-aligned square (OBB IoU=0.5 < threshold=0.6 → FP+FN) + pred = [ + Detections( + xyxy=np.array([[0, 0, 10, 10], [20, 20, 30, 30]], dtype=np.float32), + class_id=np.array([0, 1]), + confidence=np.array([0.9, 0.8], dtype=np.float32), + data={ + "xyxyxyxy": np.array( + [ + [[5, 0], [10, 5], [5, 10], [0, 5]], + [[20, 20], [30, 20], [30, 30], [20, 30]], + ], + dtype=np.float32, + ) + }, + ) + ] + + cm = ConfusionMatrix.from_detections( + predictions=pred, + targets=gt, + classes=classes, + iou_threshold=0.6, + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES, + ) + + assert cm.matrix[0, 0] == 1 # TP class 0 + assert cm.matrix[1, 2] == 1 # FN class 1 (unmatched GT) + assert cm.matrix[2, 1] == 1 # FP class 1 (unmatched pred) + + def test_confusion_matrix_metric_target_persistence_from_detections(self): + """metric_target field reflects the value passed to from_detections.""" + xyxy = np.array([[0, 0, 10, 10]], dtype=np.float32) + cm = ConfusionMatrix.from_detections( + predictions=[ + Detections( + xyxy=xyxy, class_id=np.array([0]), confidence=np.array([0.9]) + ) + ], + targets=[Detections(xyxy=xyxy, class_id=np.array([0]))], + classes=["box"], + metric_target=MetricTarget.BOXES, + ) + assert cm.metric_target == MetricTarget.BOXES + + def test_confusion_matrix_metric_target_persistence_from_tensors(self): + """metric_target field reflects the value passed to from_tensors.""" + cm = ConfusionMatrix.from_tensors( + predictions=[np.array([[0, 0, 10, 10, 0, 0.9]], dtype=np.float32)], + targets=[np.array([[0, 0, 10, 10, 0]], dtype=np.float32)], + classes=["box"], + metric_target=MetricTarget.BOXES, + ) + assert cm.metric_target == MetricTarget.BOXES