face_swapper.py

import time
import torch
import onnx
import cv2
import onnxruntime
import numpy as np
from tqdm import tqdm
import torch.nn as nn
from onnx import numpy_helper
from skimage import transform as trans
import torchvision.transforms.functional as F
import torch.nn.functional as F
from utils import mask_crop, laplacian_blending


arcface_dst = np.array(
    [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366],
     [41.5493, 92.3655], [70.7299, 92.2041]],
    dtype=np.float32)

def estimate_norm(lmk, image_size=112, mode='arcface'):
    assert lmk.shape == (5, 2)
    assert image_size % 112 == 0 or image_size % 128 == 0
    if image_size % 112 == 0:
        ratio = float(image_size) / 112.0
        diff_x = 0
    else:
        ratio = float(image_size) / 128.0
        diff_x = 8.0 * ratio
    dst = arcface_dst * ratio
    dst[:, 0] += diff_x
    tform = trans.SimilarityTransform()
    tform.estimate(lmk, dst)
    M = tform.params[0:2, :]
    return M


def norm_crop2(img, landmark, image_size=112, mode='arcface'):
    M = estimate_norm(landmark, image_size, mode)
    warped = cv2.warpAffine(img, M, (image_size, image_size), borderValue=0.0)
    return warped, M


class Inswapper():
    def __init__(self, model_file=None, batch_size=32, providers=['CPUExecutionProvider']):
        self.model_file = model_file
        self.batch_size = batch_size

        model = onnx.load(self.model_file)
        graph = model.graph
        self.emap = numpy_helper.to_array(graph.initializer[-1])

        self.session_options = onnxruntime.SessionOptions()
        self.session = onnxruntime.InferenceSession(self.model_file, sess_options=self.session_options, providers=providers)

    def forward(self, imgs, latents):
        preds = []
        for img, latent in zip(imgs, latents):
            img = img / 255
            pred = self.session.run(['output'], {'target': img, 'source': latent})[0]
            preds.append(pred)

    def get(self, imgs, target_faces, source_faces):
        imgs = list(imgs)

        preds = [None] * len(imgs)
        matrs = [None] * len(imgs)

        for idx, (img, target_face, source_face) in enumerate(zip(imgs, target_faces, source_faces)):
            matrix, blob, latent = self.prepare_data(img, target_face, source_face)
            pred = self.session.run(['output'], {'target': blob, 'source': latent})[0]
            pred = pred.transpose((0, 2, 3, 1))[0]
            pred = np.clip(255 * pred, 0, 255).astype(np.uint8)[:, :, ::-1]

            preds[idx] = pred
            matrs[idx] = matrix

        return (preds, matrs)

    def prepare_data(self, img, target_face, source_face):
        if isinstance(img, str):
            img = cv2.imread(img)

        aligned_img, matrix = norm_crop2(img, target_face.kps, 128)

        blob = cv2.dnn.blobFromImage(aligned_img, 1.0 / 255, (128, 128), (0., 0., 0.), swapRB=True)

        latent = source_face.normed_embedding.reshape((1, -1))
        latent = np.dot(latent, self.emap)
        latent /= np.linalg.norm(latent)

        return (matrix, blob, latent)

    def batch_forward(self, img_list, target_f_list, source_f_list):
        num_samples = len(img_list)
        num_batches = (num_samples + self.batch_size - 1) // self.batch_size

        for i in tqdm(range(num_batches), desc="Generating face"):
            start_idx = i * self.batch_size
            end_idx = min((i + 1) * self.batch_size, num_samples)

            batch_img = img_list[start_idx:end_idx]
            batch_target_f = target_f_list[start_idx:end_idx]
            batch_source_f = source_f_list[start_idx:end_idx]

            batch_pred, batch_matr = self.get(batch_img, batch_target_f, batch_source_f)

            yield batch_pred, batch_matr


def paste_to_whole(foreground, background, matrix, mask=None, crop_mask=(0,0,0,0), blur_amount=0.1, erode_amount = 0.15, blend_method='linear'):
    inv_matrix = cv2.invertAffineTransform(matrix)
    fg_shape = foreground.shape[:2]
    bg_shape = (background.shape[1], background.shape[0])
    foreground = cv2.warpAffine(foreground, inv_matrix, bg_shape, borderValue=0.0)

    if mask is None:
        mask = np.full(fg_shape, 1., dtype=np.float32)
        mask = mask_crop(mask, crop_mask)
        mask = cv2.warpAffine(mask, inv_matrix, bg_shape, borderValue=0.0)
    else:
        assert fg_shape == mask.shape[:2], "foreground & mask shape mismatch!"
        mask = mask_crop(mask, crop_mask).astype('float32')
        mask = cv2.warpAffine(mask, inv_matrix, (background.shape[1], background.shape[0]), borderValue=0.0)

    _mask = mask.copy()
    _mask[_mask > 0.05] = 1.
    non_zero_points = cv2.findNonZero(_mask)
    _, _, w, h = cv2.boundingRect(non_zero_points)
    mask_size = int(np.sqrt(w * h))

    if erode_amount > 0:
        kernel_size = max(int(mask_size * erode_amount), 1)
        structuring_element = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_size, kernel_size))
        mask = cv2.erode(mask, structuring_element)

    if blur_amount > 0:
        kernel_size = max(int(mask_size * blur_amount), 3)
        if kernel_size % 2 == 0:
            kernel_size += 1
        mask = cv2.GaussianBlur(mask, (kernel_size, kernel_size), 0)

    mask = np.tile(np.expand_dims(mask, axis=-1), (1, 1, 3))

    if blend_method == 'laplacian':
        composite_image = laplacian_blending(foreground, background, mask.clip(0,1), num_levels=4)
    else:
        composite_image = mask * foreground + (1 - mask) * background

    return composite_image.astype("uint8").clip(0, 255)