mmpose/visualization/simcc_vis.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

from typing import Optional, Union

import cv2 as cv
import numpy as np
import torch
from torchvision.transforms import ToPILImage


class SimCCVisualizer:

    def draw_instance_xy_heatmap(self,
                                 heatmap: torch.Tensor,
                                 overlaid_image: Optional[np.ndarray],
                                 n: int = 20,
                                 mix: bool = True,
                                 weight: float = 0.5):
        """Draw heatmaps of GT or prediction.

        Args:
            heatmap (torch.Tensor): Tensor of heatmap.
            overlaid_image (np.ndarray): The image to draw.
            n (int): Number of keypoint, up to 20.
            mix (bool):Whether to merge heatmap and original image.
            weight (float): Weight of original image during fusion.

        Returns:
            np.ndarray: the drawn image which channel is RGB.
        """
        heatmap2d = heatmap.data.max(0, keepdim=True)[0]
        xy_heatmap, K = self.split_simcc_xy(heatmap)
        K = K if K <= n else n
        blank_size = tuple(heatmap.size()[1:])
        maps = {'x': [], 'y': []}
        for i in xy_heatmap:
            x, y = self.draw_1d_heatmaps(i['x']), self.draw_1d_heatmaps(i['y'])
            maps['x'].append(x)
            maps['y'].append(y)
        white = self.creat_blank(blank_size, K)
        map2d = self.draw_2d_heatmaps(heatmap2d)
        if mix:
            map2d = cv.addWeighted(overlaid_image, 1 - weight, map2d, weight,
                                   0)
        self.image_cover(white, map2d, int(blank_size[1] * 0.1),
                         int(blank_size[0] * 0.1))
        white = self.add_1d_heatmaps(maps, white, blank_size, K)
        return white

    def split_simcc_xy(self, heatmap: Union[np.ndarray, torch.Tensor]):
        """Extract one-dimensional heatmap from two-dimensional heatmap and
        calculate the number of keypoint."""
        size = heatmap.size()
        k = size[0] if size[0] <= 20 else 20
        maps = []
        for _ in range(k):
            xy_dict = {}
            single_heatmap = heatmap[_]
            xy_dict['x'], xy_dict['y'] = self.merge_maps(single_heatmap)
            maps.append(xy_dict)
        return maps, k

    def merge_maps(self, map_2d):
        """Synthesis of one-dimensional heatmap."""
        x = map_2d.data.max(0, keepdim=True)[0]
        y = map_2d.data.max(1, keepdim=True)[0]
        return x, y

    def draw_1d_heatmaps(self, heatmap_1d):
        """Draw one-dimensional heatmap."""
        size = heatmap_1d.size()
        length = max(size)
        np_heatmap = ToPILImage()(heatmap_1d).convert('RGB')
        cv_img = cv.cvtColor(np.asarray(np_heatmap), cv.COLOR_RGB2BGR)
        if size[0] < size[1]:
            cv_img = cv.resize(cv_img, (length, 15))
        else:
            cv_img = cv.resize(cv_img, (15, length))
        single_map = cv.applyColorMap(cv_img, cv.COLORMAP_JET)
        return single_map

    def creat_blank(self,
                    size: Union[list, tuple],
                    K: int = 20,
                    interval: int = 10):
        """Create the background."""
        blank_height = int(
            max(size[0] * 2, size[0] * 1.1 + (K + 1) * (15 + interval)))
        blank_width = int(
            max(size[1] * 2, size[1] * 1.1 + (K + 1) * (15 + interval)))
        blank = np.zeros((blank_height, blank_width, 3), np.uint8)
        blank.fill(255)
        return blank

    def draw_2d_heatmaps(self, heatmap_2d):
        """Draw a two-dimensional heatmap fused with the original image."""
        np_heatmap = ToPILImage()(heatmap_2d).convert('RGB')
        cv_img = cv.cvtColor(np.asarray(np_heatmap), cv.COLOR_RGB2BGR)
        map_2d = cv.applyColorMap(cv_img, cv.COLORMAP_JET)
        return map_2d

    def image_cover(self, background: np.ndarray, foreground: np.ndarray,
                    x: int, y: int):
        """Paste the foreground on the background."""
        fore_size = foreground.shape
        background[y:y + fore_size[0], x:x + fore_size[1]] = foreground
        return background

    def add_1d_heatmaps(self,
                        maps: dict,
                        background: np.ndarray,
                        map2d_size: Union[tuple, list],
                        K: int,
                        interval: int = 10):
        """Paste one-dimensional heatmaps onto the background in turn."""
        y_startpoint, x_startpoint = [int(1.1*map2d_size[1]),
                                      int(0.1*map2d_size[0])],\
                                     [int(0.1*map2d_size[1]),
                                      int(1.1*map2d_size[0])]
        x_startpoint[1] += interval * 2
        y_startpoint[0] += interval * 2
        add = interval + 10
        for i in range(K):
            self.image_cover(background, maps['x'][i], x_startpoint[0],
                             x_startpoint[1])
            cv.putText(background, str(i),
                       (x_startpoint[0] - 30, x_startpoint[1] + 10),
                       cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2)
            self.image_cover(background, maps['y'][i], y_startpoint[0],
                             y_startpoint[1])
            cv.putText(background, str(i),
                       (y_startpoint[0], y_startpoint[1] - 5),
                       cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2)
            x_startpoint[1] += add
            y_startpoint[0] += add
        return background[:x_startpoint[1] + y_startpoint[1] +
                          1, :y_startpoint[0] + x_startpoint[0] + 1]