inferencre_video_yuv.py

import os
import cv2
import torch
import argparse
import numpy as np
from tqdm import tqdm
from torch.nn import functional as F
import warnings
import _thread
import skvideo.io
from queue import Queue, Empty
from benchmark.pytorch_msssim import ssim_matlab

warnings.filterwarnings("ignore")


def transferAudio(sourceVideo, targetVideo):
    import shutil
    import moviepy.editor
    tempAudioFileName = "./temp/audio.mkv"

    # split audio from original video file and store in "temp" directory
    if True:

        # clear old "temp" directory if it exits
        if os.path.isdir("temp"):
            # remove temp directory
            shutil.rmtree("temp")
        # create new "temp" directory
        os.makedirs("temp")
        # extract audio from video
        os.system('ffmpeg -y -i "{}" -c:a copy -vn {}'.format(sourceVideo, tempAudioFileName))

    targetNoAudio = os.path.splitext(targetVideo)[0] + "_noaudio" + os.path.splitext(targetVideo)[1]
    os.rename(targetVideo, targetNoAudio)
    # combine audio file and new video file
    os.system('ffmpeg -y -i "{}" -i {} -c copy "{}"'.format(targetNoAudio, tempAudioFileName, targetVideo))

    if os.path.getsize(
            targetVideo) == 0:  # if ffmpeg failed to merge the video and audio together try converting the audio to aac
        tempAudioFileName = "./temp/audio.m4a"
        os.system('ffmpeg -y -i "{}" -c:a aac -b:a 160k -vn {}'.format(sourceVideo, tempAudioFileName))
        os.system('ffmpeg -y -i "{}" -i {} -c copy "{}"'.format(targetNoAudio, tempAudioFileName, targetVideo))
        if (os.path.getsize(targetVideo) == 0):  # if aac is not supported by selected format
            os.rename(targetNoAudio, targetVideo)
            print("Audio transfer failed. Interpolated video will have no audio")
        else:
            print("Lossless audio transfer failed. Audio was transcoded to AAC (M4A) instead.")

            # remove audio-less video
            os.remove(targetNoAudio)
    else:
        os.remove(targetNoAudio)

    # remove temp directory
    shutil.rmtree("temp")


parser = argparse.ArgumentParser(description='Interpolation for a pair of images')
parser.add_argument('--video', dest='video', type=str, default='../../data/source/test.mp4')
parser.add_argument('--output', dest='output', type=str, default=None)
parser.add_argument('--img', dest='img', type=str, default=None)
parser.add_argument('--montage', dest='montage', action='store_true', help='montage origin video')
parser.add_argument('--model', dest='modelDir', type=str, default='train_log',
                    help='directory with trained model files')
parser.add_argument('--fp16', dest='fp16', action='store_true',
                    help='fp16 mode for faster and more lightweight inference on cards with Tensor Cores')
parser.add_argument('--UHD', dest='UHD', action='store_true', help='support 4k video')
parser.add_argument('--scale', dest='scale', type=float, default=1.0, help='Try scale=0.5 for 4k video')
parser.add_argument('--skip', dest='skip', action='store_true',
                    help='whether to remove static frames before processing')
parser.add_argument('--fps', dest='fps', type=int, default=None)
parser.add_argument('--png', dest='png', action='store_true', help='whether to vid_out png format vid_outs')
parser.add_argument('--ext', dest='ext', type=str, default='mp4', help='vid_out video extension')
parser.add_argument('--exp', dest='exp', type=int, default=1)
# Options for AdaCoF
parser.add_argument('--kernel_size', type=int, default=5)
parser.add_argument('--dilation', type=int, default=1)
args = parser.parse_args()
assert (not args.video is None or not args.img is None)
if args.UHD and args.scale == 1.0:
    args.scale = 0.5
assert args.scale in [0.25, 0.5, 1.0, 2.0, 4.0]
if not args.img is None:
    args.png = True

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.set_grad_enabled(False)
if torch.cuda.is_available():
    torch.backends.cudnn.enabled = True
    torch.backends.cudnn.benchmark = True
    if (args.fp16):
        torch.set_default_tensor_type(torch.cuda.HalfTensor)

# try:
#     from model.RIFE_HDv2 import Model
#     model = Model()
#     model.load_model(args.modelDir, -1)
#     print("Loaded v2.x HD model.")
# except:
#     from model.RIFE_HD import Model
#     model = Model()
#     model.load_model(args.modelDir, -1)
#     print("Loaded v1.x HD model")
from models.adacofnet import AdaCoFNet

model = AdaCoFNet(args)
# model = torch.nn.DataParallel(model).to(device)
# model_path = 'checkpoint/kernelsize_5/ckpt.pth'
model_path = 'checkpoint/best_checkpoints/adacof/model_epoch046.pth' # my best
# model_path = './output_adacof_train_withVimeo/checkpoint/model_epoch046.pth'
print('=========> load from: ', model_path)
checkpoint = torch.load(model_path, map_location=torch.device('cpu'))
# checkpoint = torch.load('checkpoint/model_epoch045.pth', map_location=torch.device('cpu'))
model.load_state_dict(checkpoint['state_dict'])
model.eval()
model.to(device)
# python inference_video.py --ext=yuv --exp=1 --video=./test.mp4
# ffmpeg -f rawvideo -vcodec rawvideo -s 1920x1080 -pix_fmt yuv420p -r 50 -i ./test_2X_50fps_AdaCoF.yuv -c:v libx264 -crf 12 ./test_2X_50fps_AdaCoF.mp4 

if not args.video is None:
    videoCapture = cv2.VideoCapture(args.video)  # 读视频
    fps = videoCapture.get(cv2.CAP_PROP_FPS)  # 获取视频的fps
    tot_frame = videoCapture.get(cv2.CAP_PROP_FRAME_COUNT)  # 获取视频的total frames
    videoCapture.release()  # 释放视频
    if args.fps is None:
        fpsNotAssigned = True
        args.fps = fps * (2 ** args.exp)  # fps * (2**1), fps * (2**2), fps * (2**3)
    else:
        fpsNotAssigned = False
    videogen = skvideo.io.vreader(args.video)
    lastframe = next(videogen)
    fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
    #fourcc = cv2.VideoWriter_fourcc('I', '4', '2', '0')
    video_path_wo_ext, ext = os.path.splitext(args.video)
    print('{}.{}, {} frames in total, {}FPS to {}FPS'.format(video_path_wo_ext, args.ext, tot_frame, fps, args.fps))
    if args.png == False and fpsNotAssigned == True and not args.skip:
        print("The audio will be merged after interpolation process")
    else:
        print("Will not merge audio because using png, fps or skip flag!")
else:
    videogen = []
    for f in os.listdir(args.img):
        if 'png' in f:
            videogen.append(f)
    tot_frame = len(videogen)
    videogen.sort(key=lambda x: int(x[:-4]))
    lastframe = cv2.imread(os.path.join(args.img, videogen[0]))[:, :, ::-1].copy()
    videogen = videogen[1:]
h, w, _ = lastframe.shape
vid_out_name = None
vid_out = None
if args.png:
    if not os.path.exists('vid_out'):
        os.mkdir('vid_out')
else:
    if args.output is not None:
        vid_out_name = args.output
    else:
        vid_out_name = '{}_{}X_{}fps_AdaCoF.{}'.format(video_path_wo_ext, (2 ** args.exp), int(np.round(args.fps)),
                                                       args.ext)
        print('========>', vid_out_name)
    #vid_out = cv2.VideoWriter(vid_out_name, fourcc, args.fps, (w, h))
    vid_out = open(vid_out_name, 'wb')


def clear_write_buffer(user_args, write_buffer):
    cnt = 0
    while True:
        item = write_buffer.get()
        item = cv2.cvtColor(item, cv2.COLOR_RGB2YUV_I420)
        if item is None:
            break
        if user_args.png:
            cv2.imwrite('vid_out/{:0>7d}.png'.format(cnt), item[:, :, ::-1])
            cnt += 1
        else:
            #print(item.shape)
            vid_out.write(item)


def build_read_buffer(user_args, read_buffer, videogen):
    try:
        for frame in videogen:
            if not user_args.img is None:
                frame = cv2.imread(os.path.join(user_args.img, frame))[:, :, ::-1].copy()
            if user_args.montage:
                frame = frame[:, left: left + w]
            read_buffer.put(frame)
    except:
        pass
    read_buffer.put(None)


# def make_inference(I0, I1, exp):
#     global model
#     middle = model.inference(I0, I1, args.scale)
#     if exp == 1:
#         return [middle]
#     first_half = make_inference(I0, middle, exp=exp - 1)
#     second_half = make_inference(middle, I1, exp=exp - 1)
#     return [*first_half, middle, *second_half]

def make_inference(I0, I1, exp):
    global model
    middle = model(I0, I1)
    middle = torch.clamp(middle, 0, 1)
    if exp == 1:
        return [middle]
    first_half = make_inference(I0, middle, exp=exp - 1)
    first_half = torch.clamp(first_half, 0, 1)
    second_half = make_inference(middle, I1, exp=exp - 1)
    second_half = torch.clamp(second_half, 0, 1)
    
    return [*first_half, middle, *second_half]


def pad_image(img):
    if (args.fp16):
        return F.pad(img, padding).half()
    else:
        return F.pad(img, padding)


if args.montage:
    left = w // 4
    w = w // 2
tmp = max(32, int(32 / args.scale))
ph = ((h - 1) // tmp + 1) * tmp
pw = ((w - 1) // tmp + 1) * tmp
padding = (0, pw - w, 0, ph - h)
pbar = tqdm(total=tot_frame)
skip_frame = 1
if args.montage:
    lastframe = lastframe[:, left: left + w]
write_buffer = Queue(maxsize=500)
read_buffer = Queue(maxsize=500)
_thread.start_new_thread(build_read_buffer, (args, read_buffer, videogen))
_thread.start_new_thread(clear_write_buffer, (args, write_buffer))

I1 = torch.from_numpy(np.transpose(lastframe, (2, 0, 1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
I1 = pad_image(I1)
while True:
    frame = read_buffer.get()
    if frame is None:
        break
    I0 = I1
    I1 = torch.from_numpy(np.transpose(frame, (2, 0, 1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
    I1 = pad_image(I1)
    I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False)
    I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False)
    ssim = ssim_matlab(I0_small, I1_small)
    if ssim > 0.995 and args.skip:
        if skip_frame % 100 == 0:
            print(
                "Warning: Your video has {} static frames, skipping them may change the duration of the generated video.".format(
                    skip_frame))
        skip_frame += 1
        pbar.update(1)
        continue
    if ssim < 0.5:
        output = []
        step = 1 / (2 ** args.exp)
        alpha = 0
        for i in range((2 ** args.exp) - 1):
            alpha += step
            beta = 1 - alpha
            output.append(torch.from_numpy(np.transpose(
                (cv2.addWeighted(frame[:, :, ::-1], alpha, lastframe[:, :, ::-1], beta, 0)[:, :, ::-1].copy()),
                (2, 0, 1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.)
    else:
        # images = torch.stack([I0, I0, I1, I1], dim=1)
        output = make_inference(I0, I1, args.exp)
    if args.montage:
        write_buffer.put(np.concatenate((lastframe, lastframe), 1))
        for mid in output:
            mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
            write_buffer.put(np.concatenate((lastframe, mid[:h, :w]), 1))
    else:
        write_buffer.put(lastframe)
        for mid in output:
            mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
            write_buffer.put(mid[:h, :w])
    pbar.update(1)
    lastframe = frame
if args.montage:
    write_buffer.put(np.concatenate((lastframe, lastframe), 1))
else:
    write_buffer.put(lastframe)
import time

while (not write_buffer.empty()):
    time.sleep(0.1)
pbar.close()
if not vid_out is None:
    #vid_out.release()
    vid_out.close()

# move audio to new video file if appropriate
#if args.png == False and fpsNotAssigned == True and not args.skip and not args.video is None:
#    try:
#        transferAudio(args.video, vid_out_name)
#    except:
#        print("Audio transfer failed. Interpolated video will have no audio")
#        targetNoAudio = os.path.splitext(vid_out_name)[0] + "_noaudio" + os.path.splitext(vid_out_name)[1]
#        os.rename(targetNoAudio, vid_out_name)

# python inference_video.py --exp=1 --video=../../data/source/test.mp4