lib/data/datasets/hsmr_v1/utils.py

from lib.kits.basic import *

import os
import cv2
import braceexpand
from typing import List, Union

from .crop import *


def expand_urls(urls: Union[str, List[str]]):

    def expand_url(s):
        return os.path.expanduser(os.path.expandvars(s))

    if isinstance(urls, str):
        urls = [urls]
    urls = [u for url in urls for u in braceexpand.braceexpand(expand_url(url))]
    return urls


def get_augm_args(img_augm_cfg:Optional[DictConfig]):
    '''
    Perform some random augmentation to the image and patch it. Here we perform generate augmentation arguments
    according to the configuration and random seed.

    Briefly speaking, things done here are: size scale, color scale, rotate, flip, extreme crop, translate.
    '''
    sample_args = {
            'bbox_scale'      : 1.0,
            'color_scale'     : [1.0, 1.0, 1.0],
            'rot_deg'         : 0.0,
            'do_flip'         : False,
            'do_extreme_crop' : False,
            'tx_ratio'        : 0.0,
            'ty_ratio'        : 0.0,
        }

    if img_augm_cfg is not None:
        sample_args['tx_ratio']   += np.clip(np.random.randn(), -1.0, 1.0) * img_augm_cfg.trans_factor
        sample_args['ty_ratio']   += np.clip(np.random.randn(), -1.0, 1.0) * img_augm_cfg.trans_factor
        sample_args['bbox_scale'] += np.clip(np.random.randn(), -1.0, 1.0) * img_augm_cfg.bbox_scale_factor

        if np.random.random() <= img_augm_cfg.rot_aug_rate:
            sample_args['rot_deg'] += np.clip(np.random.randn(), -2.0, 2.0) * img_augm_cfg.rot_factor
        if np.random.random() <= img_augm_cfg.flip_aug_rate:
            sample_args['do_flip'] = True
        if np.random.random() <= img_augm_cfg.extreme_crop_aug_rate:
            sample_args['do_extreme_crop'] = True

        c_up  = 1.0 + img_augm_cfg.half_color_scale
        c_low = 1.0 - img_augm_cfg.half_color_scale
        sample_args['color_scale'] = [
                np.random.uniform(c_low, c_up),
                np.random.uniform(c_low, c_up),
                np.random.uniform(c_low, c_up),
            ]
    return sample_args


def rotate_2d(pt_2d: np.ndarray, rot_rad: float) -> np.ndarray:
    '''
    Rotate a 2D point on the x-y plane.
    Copied from: https://github.com/shubham-goel/4D-Humans/blob/6ec79656a23c33237c724742ca2a0ec00b398b53/hmr2/datasets/utils.py#L90-L104

    ### Args
        - pt_2d: np.ndarray
            - Input 2D point with shape (2,).
        - rot_rad: float
            - Rotation angle.

    ### Returns
        - np.ndarray
            - Rotated 2D point.
    '''
    x = pt_2d[0]
    y = pt_2d[1]
    sn, cs = np.sin(rot_rad), np.cos(rot_rad)
    xx = x * cs - y * sn
    yy = x * sn + y * cs
    return np.array([xx, yy], dtype=np.float32)


def extreme_cropping_aggressive(center_x: float, center_y: float, width: float, height: float, keypoints_2d: np.ndarray) -> Tuple:
    """
    Perform aggressive extreme cropping.
    Copied from: https://github.com/shubham-goel/4D-Humans/blob/6ec79656a23c33237c724742ca2a0ec00b398b53/hmr2/datasets/utils.py#L978-L1025

    ### Args
        - center_x: float
            - x coordinate of bounding box center.
        - center_y: float
            - y coordinate of bounding box center.
        - width: float
            - Bounding box width.
        - height: float
            - Bounding box height.
        - keypoints_2d: np.ndarray
            - Array of shape (N, 3) containing 2D keypoint locations.
        - rescale: float
            - Scale factor to rescale bounding boxes computed from the keypoints.

    ### Returns
        - center_x: float
            - x coordinate of bounding box center.
        - center_y: float
            - y coordinate of bounding box center.
        - bbox_size: float
            - Bounding box size.
    """
    p = torch.rand(1).item()
    if full_body(keypoints_2d):
        if p < 0.2:
            center_x, center_y, width, height = crop_to_hips(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.3:
            center_x, center_y, width, height = crop_to_shoulders(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.4:
            center_x, center_y, width, height = crop_to_head(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.5:
            center_x, center_y, width, height = crop_torso_only(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.6:
            center_x, center_y, width, height = crop_rightarm_only(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.7:
            center_x, center_y, width, height = crop_leftarm_only(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.8:
            center_x, center_y, width, height = crop_legs_only(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.9:
            center_x, center_y, width, height = crop_rightleg_only(center_x, center_y, width, height, keypoints_2d)
        else:
            center_x, center_y, width, height = crop_leftleg_only(center_x, center_y, width, height, keypoints_2d)
    elif upper_body(keypoints_2d):
        if p < 0.2:
            center_x, center_y, width, height = crop_to_shoulders(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.4:
            center_x, center_y, width, height = crop_to_head(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.6:
            center_x, center_y, width, height = crop_torso_only(center_x, center_y, width, height, keypoints_2d)
        elif p < 0.8:
            center_x, center_y, width, height = crop_rightarm_only(center_x, center_y, width, height, keypoints_2d)
        else:
            center_x, center_y, width, height = crop_leftarm_only(center_x, center_y, width, height, keypoints_2d)
    return center_x, center_y, max(width, height)


def gen_trans_from_patch_cv(
    c_x        : float,
    c_y        : float,
    src_width  : float,
    src_height : float,
    dst_width  : float,
    dst_height : float,
    scale      : float,
    rot        : float
) -> np.ndarray:
    '''
    Create transformation matrix for the bounding box crop.
    Copied from: https://github.com/shubham-goel/4D-Humans/blob/6ec79656a23c33237c724742ca2a0ec00b398b53/hmr2/datasets/utils.py#L107-L154

    ### Args
        - c_x: float
            - Bounding box center x coordinate in the original image.
        - c_y: float
            - Bounding box center y coordinate in the original image.
        - src_width: float
            - Bounding box width.
        - src_height: float
            - Bounding box height.
        - dst_width: float
            - Output box width.
        - dst_height: float
            - Output box height.
        - scale: float
            - Rescaling factor for the bounding box (augmentation).
        - rot: float
            - Random rotation applied to the box.

    ### Returns
        - trans: np.ndarray
            - Target geometric transformation.
    '''
    # augment size with scale
    src_w = src_width * scale
    src_h = src_height * scale
    src_center = np.zeros(2)
    src_center[0] = c_x
    src_center[1] = c_y
    # augment rotation
    rot_rad = np.pi * rot / 180
    src_downdir = rotate_2d(np.array([0, src_h * 0.5], dtype=np.float32), rot_rad)
    src_rightdir = rotate_2d(np.array([src_w * 0.5, 0], dtype=np.float32), rot_rad)

    dst_w = dst_width
    dst_h = dst_height
    dst_center = np.array([dst_w * 0.5, dst_h * 0.5], dtype=np.float32)
    dst_downdir = np.array([0, dst_h * 0.5], dtype=np.float32)
    dst_rightdir = np.array([dst_w * 0.5, 0], dtype=np.float32)

    src = np.zeros((3, 2), dtype=np.float32)
    src[0, :] = src_center
    src[1, :] = src_center + src_downdir
    src[2, :] = src_center + src_rightdir

    dst = np.zeros((3, 2), dtype=np.float32)
    dst[0, :] = dst_center
    dst[1, :] = dst_center + dst_downdir
    dst[2, :] = dst_center + dst_rightdir

    trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))  # (2, 3)  # type: ignore

    return trans


def generate_image_patch_cv2(
    img          : np.ndarray,
    c_x          : float,
    c_y          : float,
    bb_width     : float,
    bb_height    : float,
    patch_width  : float,
    patch_height : float,
    do_flip      : bool,
    scale        : float,
    rot          : float,
    border_mode  = cv2.BORDER_CONSTANT,
    border_value = 0,
) -> Tuple[np.ndarray, np.ndarray]:
    '''
    Crop the input image and return the crop and the corresponding transformation matrix.
    Copied from: https://github.com/shubham-goel/4D-Humans/blob/6ec79656a23c33237c724742ca2a0ec00b398b53/hmr2/datasets/utils.py#L343-L386

    ### Args
        - img: np.ndarray, shape = (H, W, 3)
        - c_x: float
            - Bounding box center x coordinate in the original image.
        - c_y: float
            - Bounding box center y coordinate in the original image.
        - bb_width: float
            - Bounding box width.
        - bb_height: float
            - Bounding box height.
        - patch_width: float
            - Output box width.
        - patch_height: float
            - Output box height.
        - do_flip: bool
            - Whether to flip image or not.
        - scale: float
            - Rescaling factor for the bounding box (augmentation).
        - rot: float
            - Random rotation applied to the box.
    ### Returns
        - img_patch: np.ndarray
            - Cropped image patch of shape (patch_height, patch_height, 3)
        - trans: np.ndarray
            - Transformation matrix.
    '''
    img_height, img_width, img_channels = img.shape
    if do_flip:
        img = img[:, ::-1, :]
        c_x = img_width - c_x - 1

    trans = gen_trans_from_patch_cv(c_x, c_y, bb_width, bb_height, patch_width, patch_height, scale, rot)  # (2, 3)

    img_patch = cv2.warpAffine(img, trans, (int(patch_width), int(patch_height)),
                        flags=cv2.INTER_LINEAR,
                        borderMode=border_mode,
                        borderValue=border_value,
                )  # type: ignore
    # Force borderValue=cv2.BORDER_CONSTANT for alpha channel
    if (img.shape[2] == 4) and (border_mode != cv2.BORDER_CONSTANT):
        img_patch[:,:,3] = cv2.warpAffine(img[:,:,3], trans, (int(patch_width), int(patch_height)),
                                            flags=cv2.INTER_LINEAR,
                                            borderMode=cv2.BORDER_CONSTANT,
                            )

    return img_patch, trans


def expand_to_aspect_ratio(input_shape, target_aspect_ratio=None):
    '''
    Increase the size of the bounding box to match the target shape.
    Copied from https://github.com/shubham-goel/4D-Humans/blob/6ec79656a23c33237c724742ca2a0ec00b398b53/hmr2/datasets/utils.py#L14-L33
    '''
    if target_aspect_ratio is None:
        return input_shape

    try:
        w , h = input_shape
    except (ValueError, TypeError):
        return input_shape

    w_t, h_t = target_aspect_ratio
    if h / w < h_t / w_t:
        h_new = max(w * h_t / w_t, h)
        w_new = w
    else:
        h_new = h
        w_new = max(h * w_t / h_t, w)
    if h_new < h or w_new < w:
        breakpoint()
    return np.array([w_new, h_new])


body_permutation    = [0, 1, 5, 6, 7, 2, 3, 4, 8, 12, 13, 14, 9, 10, 11, 16, 15, 18, 17, 22, 23, 24, 19, 20, 21]
extra_permutation   = [5, 4, 3, 2, 1, 0, 11, 10, 9, 8, 7, 6, 12, 13, 14, 15, 16, 17, 18]
FLIP_KP_PERMUTATION = body_permutation + [25 + i for i in extra_permutation]

def flip_lr_keypoints(joints: np.ndarray, width: float) -> np.ndarray:
    """
    Flip 2D or 3D keypoints.
    Modified from: https://github.com/shubham-goel/4D-Humans/blob/6ec79656a23c33237c724742ca2a0ec00b398b53/hmr2/datasets/utils.py#L448-L462

    ### Args
        - joints: np.ndarray
            - Array of shape (N, 3) or (N, 4) containing 2D or 3D keypoint locations and confidence.
        - flip_permutation: list
            - Permutation to apply after flipping.
    ### Returns
        - np.ndarray
            - Flipped 2D or 3D keypoints with shape (N, 3) or (N, 4) respectively.
    """
    joints = joints.copy()
    # Flip horizontal
    joints[:, 0] = width - joints[:, 0] - 1
    joints = joints[FLIP_KP_PERMUTATION]

    return joints