utils_model.py

from utils_nms import apply_nms
from PIL import Image
import cv2
import torch
import numpy as np
import copy
import logging


def compute_boxes_and_sizes(PRED_DOWNSCALE_FACTORS, GAMMA,
                            NUM_BOXES_PER_SCALE):
    BOX_SIZE_BINS = [1]
    g_idx = 0
    while len(BOX_SIZE_BINS) < NUM_BOXES_PER_SCALE * len(
            PRED_DOWNSCALE_FACTORS):
        gamma_idx = len(BOX_SIZE_BINS) // (len(GAMMA) - 1)
        box_size = BOX_SIZE_BINS[g_idx] + GAMMA[gamma_idx]
        BOX_SIZE_BINS.append(box_size)
        g_idx += 1

    BOX_SIZE_BINS_NPY = np.array(BOX_SIZE_BINS)
    BOXES = np.reshape(BOX_SIZE_BINS_NPY, (4, 3))
    BOXES = BOXES[::-1]

    return BOXES, BOX_SIZE_BINS


def upsample_single(input_, factor=2):
    channels = input_.size(1)
    indices = torch.nonzero(input_)
    indices_up = indices.clone()
    # Corner case!
    if indices_up.size(0) == 0:
        return torch.zeros(input_.size(0),input_.size(1), input_.size(2)*factor, input_.size(3)*factor).cuda()
        # return torch.zeros(input_.size(0), input_.size(1),
        #                    input_.size(2) * factor, input_.size(3) * factor)
    indices_up[:, 2] *= factor
    indices_up[:, 3] *= factor

    output = torch.zeros(input_.size(0),input_.size(1), input_.size(2)*factor, input_.size(3)*factor).cuda()
    # output = torch.zeros(input_.size(0), input_.size(1),
    #                      input_.size(2) * factor, input_.size(3) * factor)
    output[indices_up[:, 0], indices_up[:, 1], indices_up[:, 2], indices_up[:,
                                                                 3]] = input_[
        indices[:, 0], indices[:, 1], indices[:, 2], indices[:, 3]]

    output[indices_up[:, 0], channels - 1, indices_up[:, 2] + 1, indices_up[:,
                                                                 3]] = 1.0
    output[indices_up[:, 0], channels - 1, indices_up[:, 2], indices_up[:,
                                                             3] + 1] = 1.0
    output[indices_up[:, 0], channels - 1, indices_up[:, 2] + 1, indices_up[:,
                                                                 3] + 1] = 1.0

    # output_check = nn.functional.max_pool2d(output, kernel_size=2)

    return output


def get_upsample_output(model_output, output_downscale):
    upsample_max = int(np.log2(16 // output_downscale))
    upsample_pred = []
    for idx, out in enumerate(model_output):
        out = torch.nn.functional.softmax(out, dim=1)
        upsample_out = out
        for n in range(upsample_max - idx):
            upsample_out = upsample_single(upsample_out, factor=2)
        upsample_pred.append(upsample_out.cpu().data.numpy().squeeze(0))
    return upsample_pred


def box_NMS(predictions, nms_thresh, BOXES):
    Scores = []
    Boxes = []
    for k in range(len(BOXES)):
        scores = np.max(predictions[k], axis=0)
        boxes = np.argmax(predictions[k], axis=0)
        # index the boxes with BOXES to get h_map and w_map (both are the same for us)
        mask = (boxes < 3)  # removing Z
        boxes = (boxes + 1) * mask
        scores = (
                    scores * mask)  # + 100 # added 100 since we take logsoftmax and it's negative!!

        boxes = (boxes == 1) * BOXES[k][0] + (boxes == 2) * BOXES[k][1] + (
                    boxes == 3) * BOXES[k][2]
        Scores.append(scores)
        Boxes.append(boxes)

    x, y, h, w, scores = apply_nms(Scores, Boxes, Boxes, 0.5,
                                   thresh=nms_thresh)

    nms_out = np.zeros((predictions[0].shape[1], predictions[0].shape[
        2]))  # since predictions[0] is of size 4 x H x W
    box_out = np.zeros((predictions[0].shape[1], predictions[0].shape[
        2]))  # since predictions[0] is of size 4 x H x W
    for (xx, yy, hh) in zip(x, y, h):
        nms_out[yy, xx] = 1
        box_out[yy, xx] = hh

    assert (np.count_nonzero(nms_out) == len(x))

    return nms_out, box_out


def get_box_and_dot_maps(pred, nms_thresh, BOXES):
    assert (len(pred) == 4)
    # NMS on the multi-scale outputs
    nms_out, h = box_NMS(pred, nms_thresh, BOXES)
    return nms_out, h


def get_boxed_img(image, original_emoji, h_map, w_map, gt_pred_map,
                  prediction_downscale, BOXES, BOX_SIZE_BINS,
                  thickness=1, multi_colours=False):
    if image.shape[2] != 3:
        boxed_img = image.astype(np.uint8).transpose((1, 2, 0)).copy()
    else:
        boxed_img = image.astype(np.uint8).copy()
    head_idx = np.where(gt_pred_map > 0)

    H, W = boxed_img.shape[:2]

    Y, X = head_idx[-2], head_idx[-1]

    # scale to image 
    enlarge_factor = max(((H * W) / (48 ** 2)) // 300, 1)

    for i, (y, x) in enumerate(zip(Y, X)):

        h, w = h_map[y, x] * prediction_downscale, w_map[
            y, x] * prediction_downscale
        scale = ((BOX_SIZE_BINS.index(h // prediction_downscale)) // 3) + 1

        if enlarge_factor > 1:
            h *= enlarge_factor / 2
            w *= enlarge_factor / 2

        expand_w = (0.2 * scale * w) // 2
        expand_h = (0.2 * scale * h) // 2

        y2 = min(int((prediction_downscale * x + w / 2) + expand_w), W)
        y1 = max(int((prediction_downscale * x - w / 2) - expand_w), 0)
        x2 = min(int((prediction_downscale * y + h / 2) + expand_h), H)
        x1 = max(int((prediction_downscale * y - h / 2) - expand_h), 0)

        emoji = copy.deepcopy(original_emoji)
        # emoji = original_emoji.copy()
        width = x2 - x1
        height = y2 - y1
        emoji = cv2.resize(emoji, (height, width))
        # emoji = emoji.resize((width, height))
        # emoji = np.array(emoji)

        # https://gist.github.com/clungzta/b4bbb3e2aa0490b0cfcbc042184b0b4e

        # Extract the alpha mask of the RGBA image, convert to RGB 
        r, g, b, a = cv2.split(emoji)
        overlay_color = cv2.merge((b, g, r))

        # Apply some simple filtering to remove edge noise
        mask = cv2.medianBlur(a, 5)
        mask[mask != 255] = 0
        roi = boxed_img[x1:x2, y1:y2]

        # Black-out the area behind the emoji in our original ROI
        img1_bg = cv2.bitwise_and(roi.copy(), roi.copy(),
                                  mask=cv2.bitwise_not(mask))

        # Mask out the emoji from the emoji image.
        img2_fg = cv2.bitwise_and(overlay_color, overlay_color, mask=mask)

        # Update the original image with our new ROI
        boxed_img[x1:x2, y1:y2] = cv2.add(img1_bg, img2_fg)

    return boxed_img