Spaces:
Paused
Paused
Update video_service.py
Browse files- video_service.py +181 -197
video_service.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
# video_service.py
|
| 2 |
|
|
|
|
| 3 |
import torch
|
| 4 |
import numpy as np
|
| 5 |
import random
|
|
@@ -8,15 +9,14 @@ import yaml
|
|
| 8 |
from pathlib import Path
|
| 9 |
import imageio
|
| 10 |
import tempfile
|
|
|
|
| 11 |
import sys
|
| 12 |
import subprocess
|
| 13 |
-
import threading
|
| 14 |
-
import time
|
| 15 |
-
from huggingface_hub import hf_hub_download
|
| 16 |
|
| 17 |
-
# ---
|
| 18 |
|
| 19 |
def run_setup():
|
|
|
|
| 20 |
setup_script_path = "setup.py"
|
| 21 |
if not os.path.exists(setup_script_path):
|
| 22 |
print("AVISO: script 'setup.py' não encontrado. Pulando a clonagem de dependências.")
|
|
@@ -35,6 +35,7 @@ if not LTX_VIDEO_REPO_DIR.exists():
|
|
| 35 |
run_setup()
|
| 36 |
|
| 37 |
def add_deps_to_path():
|
|
|
|
| 38 |
if not LTX_VIDEO_REPO_DIR.exists():
|
| 39 |
raise FileNotFoundError(f"Repositório LTX-Video não encontrado em '{LTX_VIDEO_REPO_DIR}'. Execute o setup.")
|
| 40 |
if str(LTX_VIDEO_REPO_DIR.resolve()) not in sys.path:
|
|
@@ -42,221 +43,204 @@ def add_deps_to_path():
|
|
| 42 |
|
| 43 |
add_deps_to_path()
|
| 44 |
|
| 45 |
-
#
|
| 46 |
from inference import (
|
| 47 |
create_ltx_video_pipeline, create_latent_upsampler,
|
| 48 |
load_image_to_tensor_with_resize_and_crop, seed_everething,
|
| 49 |
calculate_padding, load_media_file
|
| 50 |
)
|
| 51 |
-
from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem
|
| 52 |
from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
|
| 53 |
|
| 54 |
-
# ---
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
class VideoService:
|
| 61 |
def __init__(self):
|
| 62 |
-
print("Inicializando VideoService
|
| 63 |
-
self.models_loaded = False
|
| 64 |
-
self.workers = None
|
| 65 |
self.config = self._load_config()
|
| 66 |
-
self.
|
| 67 |
-
self.
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
def _load_config(self):
|
| 83 |
config_file_path = LTX_VIDEO_REPO_DIR / "configs" / "ltxv-13b-0.9.8-distilled.yaml"
|
| 84 |
with open(config_file_path, "r") as file:
|
| 85 |
return yaml.safe_load(file)
|
| 86 |
|
| 87 |
-
def
|
| 88 |
-
|
|
|
|
| 89 |
LTX_REPO = "Lightricks/LTX-Video"
|
| 90 |
-
|
| 91 |
-
self.
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
ckpt_path=self.distilled_model_path, precision=self.config["precision"],
|
| 99 |
-
text_encoder_model_name_or_path=self.config["text_encoder_model_name_or_path"],
|
| 100 |
-
sampler=self.config["sampler"], device="cpu", enhance_prompt=False,
|
| 101 |
-
prompt_enhancer_image_caption_model_name_or_path=self.config["prompt_enhancer_image_caption_model_name_or_path"],
|
| 102 |
-
prompt_enhancer_llm_model_name_or_path=self.config["prompt_enhancer_llm_model_name_or_path"],
|
| 103 |
-
)
|
| 104 |
-
latent_upsampler = create_latent_upsampler(self.spatial_upscaler_path, device="cpu")
|
| 105 |
-
pipeline.to(base_device)
|
| 106 |
-
latent_upsampler.to(upscaler_device)
|
| 107 |
return pipeline, latent_upsampler
|
| 108 |
-
|
| 109 |
-
def
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
if worker["lock"].acquire(blocking=False):
|
| 121 |
-
print(f"Worker {worker['id']} adquirido para uma nova tarefa.")
|
| 122 |
-
return worker
|
| 123 |
-
time.sleep(0.1)
|
| 124 |
-
|
| 125 |
-
def generate(self, prompt, negative_prompt, input_image_filepath=None, input_video_filepath=None,
|
| 126 |
-
height=512, width=704, mode="text-to-video", duration=2.0,
|
| 127 |
-
frames_to_use=9, seed=42, randomize_seed=True, guidance_scale=1.0, # Ignorado, mas mantido por compatibilidade
|
| 128 |
improve_texture=True, progress_callback=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
-
|
| 131 |
-
|
|
|
|
| 132 |
|
| 133 |
-
|
| 134 |
-
base_device = worker['devices']['base']
|
| 135 |
-
upscaler_device = worker['devices']['upscaler']
|
| 136 |
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
"skip_block_list": self.config["first_pass"]["skip_block_list"]
|
| 179 |
-
})
|
| 180 |
-
|
| 181 |
-
if mode == "image-to-video":
|
| 182 |
-
padding_low_res = calculate_padding(downscaled_height, downscaled_width, downscaled_height, downscaled_width)
|
| 183 |
-
media_tensor_low_res = load_image_to_tensor_with_resize_and_crop(input_image_filepath, downscaled_height, downscaled_width)
|
| 184 |
-
media_tensor_low_res = torch.nn.functional.pad(media_tensor_low_res, padding_low_res)
|
| 185 |
-
first_pass_kwargs["conditioning_items"] = [ConditioningItem(media_tensor_low_res.to(base_device), 0, 1.0)]
|
| 186 |
-
|
| 187 |
-
print(f"Worker {worker['id']}: Iniciando passe 1 em {base_device}")
|
| 188 |
-
with torch.no_grad(): low_res_latents = worker['base_pipeline'](**first_pass_kwargs).images
|
| 189 |
-
|
| 190 |
-
low_res_latents = low_res_latents.to(upscaler_device)
|
| 191 |
-
with torch.no_grad(): high_res_latents = worker['latent_upsampler'](low_res_latents)
|
| 192 |
-
high_res_latents = high_res_latents.to(base_device)
|
| 193 |
-
|
| 194 |
-
# --- PASSE 2 ---
|
| 195 |
-
second_pass_kwargs = call_kwargs_base.copy()
|
| 196 |
-
high_res_h, high_res_w = downscaled_height * 2, downscaled_width * 2
|
| 197 |
-
second_pass_kwargs.update({
|
| 198 |
-
"height": high_res_h, "width": high_res_w, "latents": high_res_latents,
|
| 199 |
-
"generator": torch.Generator(device=base_device).manual_seed(used_seed),
|
| 200 |
-
"output_type": "pt", "image_cond_noise_scale": 0.0, "guidance_scale": 1.0,
|
| 201 |
-
"timesteps": self.config["second_pass"]["timesteps"],
|
| 202 |
-
"stg_scale": self.config["second_pass"]["stg_scale"],
|
| 203 |
-
"rescaling_scale": self.config["second_pass"]["rescaling_scale"],
|
| 204 |
-
"skip_block_list": self.config["second_pass"]["skip_block_list"],
|
| 205 |
-
"tone_map_compression_ratio": self.config["second_pass"].get("tone_map_compression_ratio", 0.0)
|
| 206 |
-
})
|
| 207 |
-
|
| 208 |
-
if mode == "image-to-video":
|
| 209 |
-
padding_high_res = calculate_padding(high_res_h, high_res_w, high_res_h, high_res_w)
|
| 210 |
-
media_tensor_high_res = load_image_to_tensor_with_resize_and_crop(input_image_filepath, high_res_h, high_res_w)
|
| 211 |
-
media_tensor_high_res = torch.nn.functional.pad(media_tensor_high_res, padding_high_res)
|
| 212 |
-
second_pass_kwargs["conditioning_items"] = [ConditioningItem(media_tensor_high_res.to(base_device), 0, 1.0)]
|
| 213 |
-
|
| 214 |
-
print(f"Worker {worker['id']}: Iniciando passe 2 em {base_device}")
|
| 215 |
-
with torch.no_grad(): result_tensor = worker['base_pipeline'](**second_pass_kwargs).images
|
| 216 |
-
|
| 217 |
-
else: # Passe Único
|
| 218 |
-
single_pass_kwargs = call_kwargs_base.copy()
|
| 219 |
-
first_pass_config = self.config["first_pass"]
|
| 220 |
-
single_pass_kwargs.update({
|
| 221 |
-
"height": height_padded, "width": width_padded, "output_type": "pt",
|
| 222 |
-
"generator": torch.Generator(device=base_device).manual_seed(used_seed),
|
| 223 |
-
"guidance_scale": 1.0, **first_pass_config
|
| 224 |
-
})
|
| 225 |
-
if mode == "image-to-video":
|
| 226 |
-
media_tensor_final = load_image_to_tensor_with_resize_and_crop(input_image_filepath, height_padded, width_padded)
|
| 227 |
-
media_tensor_final = torch.nn.functional.pad(media_tensor_final, padding_values)
|
| 228 |
-
single_pass_kwargs["conditioning_items"] = [ConditioningItem(media_tensor_final.to(base_device), 0, 1.0)]
|
| 229 |
-
elif mode == "video-to-video":
|
| 230 |
-
single_pass_kwargs["media_items"] = load_media_file(media_path=input_video_filepath, height=height_padded, width=width_padded, max_frames=int(frames_to_use), padding=padding_values).to(base_device)
|
| 231 |
-
|
| 232 |
-
print(f"Worker {worker['id']}: Iniciando passe único em {base_device}")
|
| 233 |
-
with torch.no_grad(): result_tensor = worker['base_pipeline'](**single_pass_kwargs).images
|
| 234 |
-
|
| 235 |
-
if result_tensor.shape[-2:] != (height, width):
|
| 236 |
-
num_frames_final = result_tensor.shape[2]
|
| 237 |
-
videos_tensor = result_tensor.permute(0, 2, 1, 3, 4).reshape(-1, result_tensor.shape[1], result_tensor.shape[3], result_tensor.shape[4])
|
| 238 |
-
videos_resized = torch.nn.functional.interpolate(videos_tensor, size=(height, width), mode='bilinear', align_corners=False)
|
| 239 |
-
result_tensor = videos_resized.reshape(result_tensor.shape[0], num_frames_final, result_tensor.shape[1], height, width).permute(0, 2, 1, 3, 4)
|
| 240 |
-
|
| 241 |
-
result_tensor = result_tensor[:, :, :actual_num_frames, (pad_top if pad_top > 0 else None):(-pad_bottom if pad_bottom > 0 else None), (pad_left if pad_left > 0 else None):(-pad_right if pad_right > 0 else None)]
|
| 242 |
-
video_np = (result_tensor[0].permute(1, 2, 3, 0).cpu().float().numpy() * 255).astype(np.uint8)
|
| 243 |
-
temp_dir = tempfile.mkdtemp()
|
| 244 |
-
output_video_path = os.path.join(temp_dir, f"output_{used_seed}.mp4")
|
| 245 |
-
|
| 246 |
-
with imageio.get_writer(output_video_path, fps=call_kwargs_base["frame_rate"], codec='libx264', quality=8) as writer:
|
| 247 |
-
for i, frame in enumerate(video_np):
|
| 248 |
-
writer.append_data(frame)
|
| 249 |
-
if progress_callback: progress_callback(i + 1, len(video_np))
|
| 250 |
-
return output_video_path, used_seed
|
| 251 |
-
|
| 252 |
-
except Exception as e:
|
| 253 |
-
print(f"!!!!!!!! ERRO no Worker {worker['id']} !!!!!!!!\n{e}")
|
| 254 |
-
raise e
|
| 255 |
-
finally:
|
| 256 |
-
print(f"Worker {worker['id']}: Tarefa finalizada. Limpando cache e liberando worker...")
|
| 257 |
-
with torch.cuda.device(base_device): torch.cuda.empty_cache()
|
| 258 |
-
with torch.cuda.device(upscaler_device): torch.cuda.empty_cache()
|
| 259 |
-
worker["lock"].release()
|
| 260 |
|
| 261 |
-
|
| 262 |
video_generation_service = VideoService()
|
|
|
|
| 1 |
# video_service.py
|
| 2 |
|
| 3 |
+
# --- 1. IMPORTAÇÕES ---
|
| 4 |
import torch
|
| 5 |
import numpy as np
|
| 6 |
import random
|
|
|
|
| 9 |
from pathlib import Path
|
| 10 |
import imageio
|
| 11 |
import tempfile
|
| 12 |
+
from huggingface_hub import hf_hub_download
|
| 13 |
import sys
|
| 14 |
import subprocess
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
# --- 2. GERENCIAMENTO DE DEPENDÊNCIAS E SETUP ---
|
| 17 |
|
| 18 |
def run_setup():
|
| 19 |
+
"""Executa o script setup.py para clonar as dependências necessárias."""
|
| 20 |
setup_script_path = "setup.py"
|
| 21 |
if not os.path.exists(setup_script_path):
|
| 22 |
print("AVISO: script 'setup.py' não encontrado. Pulando a clonagem de dependências.")
|
|
|
|
| 35 |
run_setup()
|
| 36 |
|
| 37 |
def add_deps_to_path():
|
| 38 |
+
"""Adiciona o repositório clonado ao sys.path para que suas bibliotecas possam ser importadas."""
|
| 39 |
if not LTX_VIDEO_REPO_DIR.exists():
|
| 40 |
raise FileNotFoundError(f"Repositório LTX-Video não encontrado em '{LTX_VIDEO_REPO_DIR}'. Execute o setup.")
|
| 41 |
if str(LTX_VIDEO_REPO_DIR.resolve()) not in sys.path:
|
|
|
|
| 43 |
|
| 44 |
add_deps_to_path()
|
| 45 |
|
| 46 |
+
# --- 3. IMPORTAÇÕES ESPECÍFICAS DO MODELO ---
|
| 47 |
from inference import (
|
| 48 |
create_ltx_video_pipeline, create_latent_upsampler,
|
| 49 |
load_image_to_tensor_with_resize_and_crop, seed_everething,
|
| 50 |
calculate_padding, load_media_file
|
| 51 |
)
|
| 52 |
+
from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem, LTXMultiScalePipeline
|
| 53 |
from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
|
| 54 |
|
| 55 |
+
# --- 4. FUNÇÕES HELPER DE LOG ---
|
| 56 |
+
def log_tensor_info(tensor, name="Tensor"):
|
| 57 |
+
if not isinstance(tensor, torch.Tensor):
|
| 58 |
+
print(f"\n[INFO] O item '{name}' não é um tensor para logar.")
|
| 59 |
+
return
|
| 60 |
+
print(f"\n--- Informações do Tensor: {name} ---")
|
| 61 |
+
print(f" - Shape: {tensor.shape}")
|
| 62 |
+
print(f" - Dtype: {tensor.dtype}")
|
| 63 |
+
print(f" - Device: {tensor.device}")
|
| 64 |
+
if tensor.numel() > 0:
|
| 65 |
+
print(f" - Min valor: {tensor.min().item():.4f}")
|
| 66 |
+
print(f" - Max valor: {tensor.max().item():.4f}")
|
| 67 |
+
print(f" - Média: {tensor.mean().item():.4f}")
|
| 68 |
+
else:
|
| 69 |
+
print(" - O tensor está vazio, sem estatísticas.")
|
| 70 |
+
print("------------------------------------------\n")
|
| 71 |
+
|
| 72 |
+
# --- 5. CLASSE PRINCIPAL DO SERVIÇO ---
|
| 73 |
class VideoService:
|
| 74 |
def __init__(self):
|
| 75 |
+
print("Inicializando VideoService...")
|
|
|
|
|
|
|
| 76 |
self.config = self._load_config()
|
| 77 |
+
self.device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 78 |
+
self.last_memory_reserved_mb = 0
|
| 79 |
+
self.pipeline, self.latent_upsampler = self._load_models()
|
| 80 |
+
print(f"Movendo modelos para o dispositivo de inferência: {self.device}")
|
| 81 |
+
self.pipeline.to(self.device)
|
| 82 |
+
if self.latent_upsampler:
|
| 83 |
+
self.latent_upsampler.to(self.device)
|
| 84 |
+
if self.device == "cuda":
|
| 85 |
+
torch.cuda.empty_cache()
|
| 86 |
+
self._log_gpu_memory("Após carregar modelos")
|
| 87 |
+
print("VideoService pronto para uso.")
|
| 88 |
+
|
| 89 |
+
def _log_gpu_memory(self, stage_name: str):
|
| 90 |
+
if self.device != "cuda": return
|
| 91 |
+
current_reserved_b = torch.cuda.memory_reserved()
|
| 92 |
+
current_reserved_mb = current_reserved_b / (1024 ** 2)
|
| 93 |
+
total_memory_b = torch.cuda.get_device_properties(0).total_memory
|
| 94 |
+
total_memory_mb = total_memory_b / (1024 ** 2)
|
| 95 |
+
peak_reserved_mb = torch.cuda.max_memory_reserved() / (1024 ** 2)
|
| 96 |
+
delta_mb = current_reserved_mb - self.last_memory_reserved_mb
|
| 97 |
+
print(f"\n--- [LOG DE MEMÓRIA GPU] - {stage_name} ---")
|
| 98 |
+
print(f" - Uso Atual (Reservado): {current_reserved_mb:.2f} MB / {total_memory_mb:.2f} MB")
|
| 99 |
+
print(f" - Variação desde o último log: {delta_mb:+.2f} MB")
|
| 100 |
+
if peak_reserved_mb > self.last_memory_reserved_mb:
|
| 101 |
+
print(f" - Pico de Uso (nesta operação): {peak_reserved_mb:.2f} MB")
|
| 102 |
+
print("--------------------------------------------------\n")
|
| 103 |
+
self.last_memory_reserved_mb = current_reserved_mb
|
| 104 |
|
| 105 |
def _load_config(self):
|
| 106 |
config_file_path = LTX_VIDEO_REPO_DIR / "configs" / "ltxv-13b-0.9.8-distilled.yaml"
|
| 107 |
with open(config_file_path, "r") as file:
|
| 108 |
return yaml.safe_load(file)
|
| 109 |
|
| 110 |
+
def _load_models(self):
|
| 111 |
+
models_dir = "downloaded_models_gradio"
|
| 112 |
+
Path(models_dir).mkdir(parents=True, exist_ok=True)
|
| 113 |
LTX_REPO = "Lightricks/LTX-Video"
|
| 114 |
+
distilled_model_path = hf_hub_download(repo_id=LTX_REPO, filename=self.config["checkpoint_path"], local_dir=models_dir, local_dir_use_symlinks=False)
|
| 115 |
+
self.config["checkpoint_path"] = distilled_model_path
|
| 116 |
+
spatial_upscaler_path = hf_hub_download(repo_id=LTX_REPO, filename=self.config["spatial_upscaler_model_path"], local_dir=models_dir, local_dir_use_symlinks=False)
|
| 117 |
+
self.config["spatial_upscaler_model_path"] = spatial_upscaler_path
|
| 118 |
+
pipeline = create_ltx_video_pipeline(ckpt_path=self.config["checkpoint_path"], precision=self.config["precision"], text_encoder_model_name_or_path=self.config["text_encoder_model_name_or_path"], sampler=self.config["sampler"], device="cpu", enhance_prompt=False, prompt_enhancer_image_caption_model_name_or_path=self.config["prompt_enhancer_image_caption_model_name_or_path"], prompt_enhancer_llm_model_name_or_path=self.config["prompt_enhancer_llm_model_name_or_path"])
|
| 119 |
+
latent_upsampler = None
|
| 120 |
+
if self.config.get("spatial_upscaler_model_path"):
|
| 121 |
+
latent_upsampler = create_latent_upsampler(self.config["spatial_upscaler_model_path"], device="cpu")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
return pipeline, latent_upsampler
|
| 123 |
+
|
| 124 |
+
def _prepare_conditioning_tensor(self, filepath, height, width, padding_values):
|
| 125 |
+
tensor = load_image_to_tensor_with_resize_and_crop(filepath, height, width)
|
| 126 |
+
tensor = torch.nn.functional.pad(tensor, padding_values)
|
| 127 |
+
return tensor.to(self.device)
|
| 128 |
+
|
| 129 |
+
def generate(self, prompt, negative_prompt, mode="text-to-video",
|
| 130 |
+
start_image_filepath=None,
|
| 131 |
+
middle_image_filepath=None, middle_frame_number=None, middle_image_weight=1.0,
|
| 132 |
+
end_image_filepath=None, end_image_weight=1.0,
|
| 133 |
+
input_video_filepath=None, height=512, width=704, duration=2.0,
|
| 134 |
+
frames_to_use=9, seed=42, randomize_seed=True, guidance_scale=3.0,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
improve_texture=True, progress_callback=None):
|
| 136 |
+
if self.device == "cuda":
|
| 137 |
+
torch.cuda.empty_cache()
|
| 138 |
+
torch.cuda.reset_peak_memory_stats()
|
| 139 |
+
self._log_gpu_memory("Início da Geração")
|
| 140 |
+
|
| 141 |
+
if mode == "image-to-video" and not start_image_filepath:
|
| 142 |
+
raise ValueError("A imagem de início é obrigatória para o modo image-to-video")
|
| 143 |
+
if mode == "video-to-video" and not input_video_filepath:
|
| 144 |
+
raise ValueError("O vídeo de entrada é obrigatório para o modo video-to-video")
|
| 145 |
+
|
| 146 |
+
used_seed = random.randint(0, 2**32 - 1) if randomize_seed else int(seed)
|
| 147 |
+
seed_everething(used_seed)
|
| 148 |
+
|
| 149 |
+
FPS = 24.0
|
| 150 |
+
MAX_NUM_FRAMES = 257
|
| 151 |
+
target_frames_rounded = round(duration * FPS)
|
| 152 |
+
n_val = round((float(target_frames_rounded) - 1.0) / 8.0)
|
| 153 |
+
actual_num_frames = max(9, min(MAX_NUM_FRAMES, int(n_val * 8 + 1)))
|
| 154 |
|
| 155 |
+
height_padded = ((height - 1) // 32 + 1) * 32
|
| 156 |
+
width_padded = ((width - 1) // 32 + 1) * 32
|
| 157 |
+
padding_values = calculate_padding(height, width, height_padded, width_padded)
|
| 158 |
|
| 159 |
+
generator = torch.Generator(device=self.device).manual_seed(used_seed)
|
|
|
|
|
|
|
| 160 |
|
| 161 |
+
conditioning_items = []
|
| 162 |
+
if mode == "image-to-video":
|
| 163 |
+
start_tensor = self._prepare_conditioning_tensor(start_image_filepath, height, width, padding_values)
|
| 164 |
+
conditioning_items.append(ConditioningItem(start_tensor, 0, 1.0))
|
| 165 |
+
if middle_image_filepath and middle_frame_number is not None:
|
| 166 |
+
middle_tensor = self._prepare_conditioning_tensor(middle_image_filepath, height, width, padding_values)
|
| 167 |
+
safe_middle_frame = max(0, min(int(middle_frame_number), actual_num_frames - 1))
|
| 168 |
+
conditioning_items.append(ConditioningItem(middle_tensor, safe_middle_frame, float(middle_image_weight)))
|
| 169 |
+
if end_image_filepath:
|
| 170 |
+
end_tensor = self._prepare_conditioning_tensor(end_image_filepath, height, width, padding_values)
|
| 171 |
+
last_frame_index = actual_num_frames - 1
|
| 172 |
+
conditioning_items.append(ConditioningItem(end_tensor, last_frame_index, float(end_image_weight)))
|
| 173 |
+
|
| 174 |
+
call_kwargs = {
|
| 175 |
+
"prompt": prompt, "negative_prompt": negative_prompt, "height": height_padded, "width": width_padded,
|
| 176 |
+
"num_frames": actual_num_frames, "frame_rate": int(FPS), "generator": generator, "output_type": "pt",
|
| 177 |
+
"conditioning_items": conditioning_items if conditioning_items else None,
|
| 178 |
+
"media_items": None,
|
| 179 |
+
"decode_timestep": self.config["decode_timestep"], "decode_noise_scale": self.config["decode_noise_scale"],
|
| 180 |
+
"stochastic_sampling": self.config["stochastic_sampling"], "image_cond_noise_scale": 0.15,
|
| 181 |
+
"is_video": True, "vae_per_channel_normalize": True,
|
| 182 |
+
"mixed_precision": (self.config["precision"] == "mixed_precision"),
|
| 183 |
+
"offload_to_cpu": False, "enhance_prompt": False,
|
| 184 |
+
"skip_layer_strategy": SkipLayerStrategy.AttentionValues
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
if mode == "video-to-video":
|
| 188 |
+
call_kwargs["media_items"] = load_media_file(media_path=input_video_filepath, height=height, width=width, max_frames=int(frames_to_use), padding=padding_values).to(self.device)
|
| 189 |
+
|
| 190 |
+
result_tensor = None
|
| 191 |
+
if improve_texture:
|
| 192 |
+
if not self.latent_upsampler:
|
| 193 |
+
raise ValueError("Upscaler espacial não carregado.")
|
| 194 |
+
multi_scale_pipeline = LTXMultiScalePipeline(self.pipeline, self.latent_upsampler)
|
| 195 |
+
first_pass_args = self.config.get("first_pass", {}).copy()
|
| 196 |
+
first_pass_args["guidance_scale"] = float(guidance_scale)
|
| 197 |
+
second_pass_args = self.config.get("second_pass", {}).copy()
|
| 198 |
+
second_pass_args["guidance_scale"] = float(guidance_scale)
|
| 199 |
+
multi_scale_call_kwargs = call_kwargs.copy()
|
| 200 |
+
multi_scale_call_kwargs.update({"downscale_factor": self.config["downscale_factor"], "first_pass": first_pass_args, "second_pass": second_pass_args})
|
| 201 |
+
result_tensor = multi_scale_pipeline(**multi_scale_call_kwargs).images
|
| 202 |
+
log_tensor_info(result_tensor, "Resultado da Etapa 2 (Saída do Pipeline Multi-Scale)")
|
| 203 |
+
else:
|
| 204 |
+
single_pass_kwargs = call_kwargs.copy()
|
| 205 |
+
first_pass_config = self.config.get("first_pass", {})
|
| 206 |
+
single_pass_kwargs.update({
|
| 207 |
+
"guidance_scale": float(guidance_scale),
|
| 208 |
+
"stg_scale": first_pass_config.get("stg_scale"),
|
| 209 |
+
"rescaling_scale": first_pass_config.get("rescaling_scale"),
|
| 210 |
+
"skip_block_list": first_pass_config.get("skip_block_list"),
|
| 211 |
+
})
|
| 212 |
+
|
| 213 |
+
# --- <INÍCIO DA CORREÇÃO> ---
|
| 214 |
+
if mode == "video-to-video":
|
| 215 |
+
single_pass_kwargs["timesteps"] = [0.7] # CORRIGIDO: Passar como uma lista
|
| 216 |
+
print("[INFO] Modo video-to-video (etapa única): definindo timesteps (força) para [0.7]")
|
| 217 |
+
else:
|
| 218 |
+
single_pass_kwargs["timesteps"] = first_pass_config.get("timesteps")
|
| 219 |
+
# --- <FIM DA CORREÇÃO> ---
|
| 220 |
|
| 221 |
+
print("\n[INFO] Executando pipeline de etapa única...")
|
| 222 |
+
result_tensor = self.pipeline(**single_pass_kwargs).images
|
| 223 |
+
|
| 224 |
+
pad_left, pad_right, pad_top, pad_bottom = padding_values
|
| 225 |
+
slice_h_end = -pad_bottom if pad_bottom > 0 else None
|
| 226 |
+
slice_w_end = -pad_right if pad_right > 0 else None
|
| 227 |
+
|
| 228 |
+
result_tensor = result_tensor[:, :, :actual_num_frames, pad_top:slice_h_end, pad_left:slice_w_end]
|
| 229 |
+
log_tensor_info(result_tensor, "Tensor Final (Após Pós-processamento, Antes de Salvar)")
|
| 230 |
+
|
| 231 |
+
video_np = (result_tensor[0].permute(1, 2, 3, 0).cpu().float().numpy() * 255).astype(np.uint8)
|
| 232 |
+
temp_dir = tempfile.mkdtemp()
|
| 233 |
+
output_video_path = os.path.join(temp_dir, f"output_{used_seed}.mp4")
|
| 234 |
+
|
| 235 |
+
with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], codec='libx264', quality=8) as writer:
|
| 236 |
+
total_frames = len(video_np)
|
| 237 |
+
for i, frame in enumerate(video_np):
|
| 238 |
+
writer.append_data(frame)
|
| 239 |
+
if progress_callback:
|
| 240 |
+
progress_callback(i + 1, total_frames)
|
| 241 |
+
|
| 242 |
+
self._log_gpu_memory("Fim da Geração")
|
| 243 |
+
return output_video_path, used_seed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
|
| 245 |
+
print("Criando instância do VideoService. O carregamento do modelo começará agora...")
|
| 246 |
video_generation_service = VideoService()
|