mmscan_utils/box_metric.py

from typing import Dict, Tuple, Union

import numpy as np
import torch
from scipy.optimize import linear_sum_assignment


def average_precision(recalls: np.ndarray,
                      precisions: np.ndarray,
                      mode: str = 'area') -> np.ndarray:
    """Calculate average precision (for single or multiple scales).

    Args:
        recalls (np.ndarray): Recalls with shape of (num_scales, num_dets)
            or (num_dets, ).
        precisions (np.ndarray): Precisions with shape of
            (num_scales, num_dets) or (num_dets, ).
        mode (str): 'area' or '11points', 'area' means calculating the area
            under precision-recall curve, '11points' means calculating
            the average precision of recalls at [0, 0.1, ..., 1]
            Defaults to 'area'.

    Returns:
        np.ndarray: Calculated average precision.
    """
    if recalls.ndim == 1:
        recalls = recalls[np.newaxis, :]
        precisions = precisions[np.newaxis, :]

    assert recalls.shape == precisions.shape
    assert recalls.ndim == 2

    num_scales = recalls.shape[0]
    ap = np.zeros(num_scales, dtype=np.float32)

    if mode == 'area':
        zeros = np.zeros((num_scales, 1), dtype=recalls.dtype)
        ones = np.ones((num_scales, 1), dtype=recalls.dtype)
        mrec = np.hstack((zeros, recalls, ones))
        mpre = np.hstack((zeros, precisions, zeros))
        for i in range(mpre.shape[1] - 1, 0, -1):
            mpre[:, i - 1] = np.maximum(mpre[:, i - 1], mpre[:, i])
        for i in range(num_scales):
            ind = np.where(mrec[i, 1:] != mrec[i, :-1])[0]
            ap[i] = np.sum(
                (mrec[i, ind + 1] - mrec[i, ind]) * mpre[i, ind + 1])

    elif mode == '11points':
        for i in range(num_scales):
            for thr in np.arange(0, 1 + 1e-3, 0.1):
                precs = precisions[i, recalls[i, :] >= thr]
                prec = precs.max() if precs.size > 0 else 0
                ap[i] += prec
            ap /= 11
    else:
        raise ValueError(
            'Unrecognized mode, only "area" and "11points" are supported')
    return ap


def get_f1_scores(iou_matrix: Union[np.ndarray, torch.tensor],
                  iou_threshold) -> float:
    """Refer to the algorithm in Multi3DRefer to compute the F1 score.

    Args:
        iou_matrix (ndarray/tensor):
            The iou matrix of the predictions and ground truths with
                shape (num_preds , num_gts)
        iou_threshold (float): 0.25/0.5

    Returns:
        float: the f1 score as the result
    """
    iou_thr_tp = 0
    pred_bboxes_count, gt_bboxes_count = iou_matrix.shape

    square_matrix_len = max(gt_bboxes_count, pred_bboxes_count)
    iou_matrix_fill = np.zeros(shape=(square_matrix_len, square_matrix_len),
                               dtype=np.float32)
    iou_matrix_fill[:pred_bboxes_count, :gt_bboxes_count] = iou_matrix

    # apply matching algorithm
    row_idx, col_idx = linear_sum_assignment(iou_matrix_fill * -1)

    # iterate matched pairs, check ious
    for i in range(pred_bboxes_count):
        iou = iou_matrix[row_idx[i], col_idx[i]]
        # calculate true positives
        if iou >= iou_threshold:
            iou_thr_tp += 1

    # calculate precision, recall and f1-score for the current scene
    f1_score = 2 * iou_thr_tp / (pred_bboxes_count + gt_bboxes_count)

    return f1_score


def __get_fp_tp_array__(iou_array: Union[np.ndarray, torch.tensor],
                        iou_threshold: float) \
                        -> Tuple[np.ndarray, np.ndarray]:
    """Compute the False-positive and True-positive array for each prediction.

    Args:
        iou_array (ndarray/tensor):
            the iou matrix of the predictions and ground truths
            (shape num_preds, num_gts)
        iou_threshold (float): 0.25/0.5

    Returns:
        np.ndarray, np.ndarray: (len(preds)),
        the false-positive and true-positive array for each prediction.
    """
    gt_matched_records = np.zeros((len(iou_array[0])), dtype=bool)
    tp_thr = np.zeros((len(iou_array)))
    fp_thr = np.zeros((len(iou_array)))

    for d, _ in enumerate(range(len(iou_array))):
        iou_max = -np.inf
        cur_iou = iou_array[d]
        num_gts = cur_iou.shape[0]

        if num_gts > 0:
            for j in range(num_gts):
                iou = cur_iou[j]
                if iou > iou_max:
                    iou_max = iou
                    jmax = j

        if iou_max >= iou_threshold:
            if not gt_matched_records[jmax]:
                gt_matched_records[jmax] = True
                tp_thr[d] = 1.0
            else:
                fp_thr[d] = 1.0
        else:
            fp_thr[d] = 1.0

    return fp_thr, tp_thr


def subset_get_average_precision(subset_results: dict,
                                 iou_thr: float)\
                                 -> Tuple[np.ndarray, np.ndarray]:
    """Return the average precision and max recall for a given iou array,
    "subset" version while the num_gt of each sample may differ.

    Args:
        subset_results (dict):
            The results, consisting of scores, sample_indices, ious.
            sample_indices means which sample the prediction belongs to.
        iou_threshold (float): 0.25/0.5

    Returns:
        Tuple[np.ndarray, np.ndarray]: the average precision and max recall.
    """
    confidences = subset_results['scores']
    sample_indices = subset_results['sample_indices']
    ious = subset_results['ious']
    gt_matched_records = {}
    total_gt_boxes = 0
    for i, sample_idx in enumerate(sample_indices):
        if sample_idx not in gt_matched_records:
            gt_matched_records[sample_idx] = np.zeros((len(ious[i]), ),
                                                      dtype=bool)
            total_gt_boxes += ious[i].shape[0]

    confidences = np.array(confidences)
    sorted_inds = np.argsort(-confidences)
    sample_indices = [sample_indices[i] for i in sorted_inds]
    ious = [ious[i] for i in sorted_inds]

    tp_thr = np.zeros(len(sample_indices))
    fp_thr = np.zeros(len(sample_indices))

    for d, sample_idx in enumerate(sample_indices):
        iou_max = -np.inf
        cur_iou = ious[d]
        num_gts = cur_iou.shape[0]
        if num_gts > 0:
            for j in range(num_gts):
                iou = cur_iou[j]
                if iou > iou_max:
                    iou_max = iou
                    jmax = j

        if iou_max >= iou_thr:
            if not gt_matched_records[sample_idx][jmax]:
                gt_matched_records[sample_idx][jmax] = True
                tp_thr[d] = 1.0
            else:
                fp_thr[d] = 1.0
        else:
            fp_thr[d] = 1.0

    fp = np.cumsum(fp_thr)
    tp = np.cumsum(tp_thr)
    recall = tp / float(total_gt_boxes)
    precision = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)

    return average_precision(recall, precision), np.max(recall)


def get_average_precision(iou_array: np.ndarray, iou_threshold: float) \
        -> Tuple[np.ndarray, np.ndarray]:
    """Return the average precision and max recall for a given iou array.

    Args:
        iou_array (ndarray/tensor):
            The iou matrix of the predictions and ground truths
            (shape len(preds)*len(gts))
        iou_threshold (float): 0.25/0.5

    Returns:
        Tuple[np.ndarray, np.ndarray]: the average precision and max recall.
    """
    fp, tp = __get_fp_tp_array__(iou_array, iou_threshold)
    fp_cum = np.cumsum(fp)
    tp_cum = np.cumsum(tp)
    recall = tp_cum / float(iou_array.shape[1])
    precision = tp_cum / np.maximum(tp_cum + fp_cum, np.finfo(np.float64).eps)

    return average_precision(recall, precision), np.max(recall)


def get_general_topk_scores(iou_array: Union[np.ndarray, torch.tensor],
                            iou_threshold: float,
                            mode: str = 'sigma') -> Dict[str, float]:
    """Compute the multi-topk metric, we provide two modes.

    Args:
        iou_array (ndarray/tensor):
            the iou matrix of the predictions and ground truths
            (shape len(preds)*len(gts))
        iou_threshold (float): 0.25/0.5
        mode (str): 'sigma'/'simple'
                "simple": 1/N * Hit(min(N*k,len(pred)))
                "sigma": 1/N * Sigma [Hit(min(n*k,len(pred)))>=n] n = 1~N
                    Hit(M) return the number of gtound truths hitted by
                    the first M predictions.
                    N = the number of gtound truths
                Default to 'sigma'.

    Returns:
        Dict[str,float]: the score of multi-topk metric.
    """

    assert mode in ['sigma', 'simple']
    topk_scores = []
    gt_matched_records = np.zeros(len(iou_array[0]))
    num_gt = len(gt_matched_records)
    for d, _ in enumerate(range(len(iou_array))):
        iou_max = -np.inf
        cur_iou = iou_array[d]

        for j in range(len(iou_array[d])):
            iou = cur_iou[j]
            if iou > iou_max:
                iou_max = iou
                j_max = j
        if iou_max >= iou_threshold:
            gt_matched_records[j_max] = True
        topk_scores.append(gt_matched_records.copy())

    topk_results = {}
    for topk in [1, 3, 5, 10]:
        if mode == 'sigma':
            scores = [
                int(
                    np.sum(topk_scores[min(n * topk, len(topk_scores)) -
                                       1]) >= n) for n in range(1, num_gt + 1)
            ]
            result = np.sum(scores) / num_gt
        else:
            query_index = min(num_gt * topk, len(topk_scores)) - 1
            result = np.sum(topk_scores[query_index]) / num_gt
        topk_results[f'gTop-{topk}@{iou_threshold}'] = result
    return topk_results