video_diffusion/zero_shot/zero_shot_text2video.py

import gradio as gr
import imageio
import torch
from diffusers import TextToVideoZeroPipeline

from video_diffusion.tuneavideo.util import save_videos_grid
from video_diffusion.utils.model_list import stable_model_list


class ZeroShotText2VideoGenerator:
    def __init__(self):
        self.pipe = None

    def load_model(self, model_id):
        if self.pipe is None:
            self.pipe = TextToVideoZeroPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda")
            self.pipe.to("cuda")
            self.pipe.enable_xformers_memory_efficient_attention()
            self.pipe.enable_attention_slicing()

        return self.pipe

    def generate_video(
        self,
        prompt,
        negative_prompt,
        model_id,
        height,
        width,
        video_length,
        guidance_scale,
        fps,
        t0,
        t1,
        motion_field_strength_x,
        motion_field_strength_y,
    ):
        pipe = self.load_model(model_id)
        result = pipe(
            prompt=prompt,
            negative_prompt=negative_prompt,
            height=height,
            width=width,
            video_length=video_length,
            guidance_scale=guidance_scale,
            t0=t0,
            t1=t1,
            motion_field_strength_x=motion_field_strength_x,
            motion_field_strength_y=motion_field_strength_y,
        ).images

        result = [(r * 255).astype("uint8") for r in result]
        imageio.mimsave("video.mp4", result, fps=fps)
        return "video.mp4"

    def app():
        with gr.Blocks():
            with gr.Row():
                with gr.Column():
                    zero_shot_text2video_prompt = gr.Textbox(
                        lines=1,
                        placeholder="Prompt",
                        show_label=False,
                    )
                    zero_shot_text2video_negative_prompt = gr.Textbox(
                        lines=1,
                        placeholder="Negative Prompt",
                        show_label=False,
                    )
                    zero_shot_text2video_model_id = gr.Dropdown(
                        choices=stable_model_list,
                        label="Stable Model List",
                        value=stable_model_list[0],
                    )
                    with gr.Row():
                        with gr.Column():
                            zero_shot_text2video_guidance_scale = gr.Slider(
                                label="Guidance Scale",
                                minimum=1,
                                maximum=15,
                                step=1,
                                value=7.5,
                            )
                            zero_shot_text2video_video_length = gr.Slider(
                                label="Video Length",
                                minimum=1,
                                maximum=100,
                                step=1,
                                value=10,
                            )
                            zero_shot_text2video_t0 = gr.Slider(
                                label="Timestep T0",
                                minimum=0,
                                maximum=100,
                                step=1,
                                value=44,
                            )
                            zero_shot_text2video_motion_field_strength_x = gr.Slider(
                                label="Motion Field Strength X",
                                minimum=0,
                                maximum=100,
                                step=1,
                                value=12,
                            )
                            zero_shot_text2video_fps = gr.Slider(
                                label="Fps",
                                minimum=1,
                                maximum=60,
                                step=1,
                                value=10,
                            )
                        with gr.Row():
                            with gr.Column():
                                zero_shot_text2video_height = gr.Slider(
                                    label="Height",
                                    minimum=128,
                                    maximum=1280,
                                    step=32,
                                    value=512,
                                )
                                zero_shot_text2video_width = gr.Slider(
                                    label="Width",
                                    minimum=128,
                                    maximum=1280,
                                    step=32,
                                    value=512,
                                )
                                zero_shot_text2video_t1 = gr.Slider(
                                    label="Timestep T1",
                                    minimum=0,
                                    maximum=100,
                                    step=1,
                                    value=47,
                                )
                                zero_shot_text2video_motion_field_strength_y = gr.Slider(
                                    label="Motion Field Strength Y",
                                    minimum=0,
                                    maximum=100,
                                    step=1,
                                    value=12,
                                )
                    zero_shot_text2video_button = gr.Button(value="Generator")

                with gr.Column():
                    zero_shot_text2video_output = gr.Video(label="Output")

            zero_shot_text2video_button.click(
                fn=ZeroShotText2VideoGenerator().generate_video,
                inputs=[
                    zero_shot_text2video_prompt,
                    zero_shot_text2video_negative_prompt,
                    zero_shot_text2video_model_id,
                    zero_shot_text2video_height,
                    zero_shot_text2video_width,
                    zero_shot_text2video_video_length,
                    zero_shot_text2video_guidance_scale,
                    zero_shot_text2video_fps,
                    zero_shot_text2video_t0,
                    zero_shot_text2video_t1,
                    zero_shot_text2video_motion_field_strength_x,
                    zero_shot_text2video_motion_field_strength_y,
                ],
                outputs=zero_shot_text2video_output,
            )