Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,462 +1,307 @@
|
|
| 1 |
-
# FILE:
|
| 2 |
-
# DESCRIPTION:
|
| 3 |
-
#
|
| 4 |
|
| 5 |
-
import
|
| 6 |
-
import io
|
| 7 |
-
import json
|
| 8 |
-
import logging
|
| 9 |
-
import os
|
| 10 |
-
import random
|
| 11 |
-
import shutil
|
| 12 |
-
import subprocess
|
| 13 |
-
import sys
|
| 14 |
-
import tempfile
|
| 15 |
-
import time
|
| 16 |
import traceback
|
| 17 |
-
import
|
| 18 |
-
from pathlib import Path
|
| 19 |
-
from typing import Dict, List, Optional, Tuple
|
| 20 |
-
|
| 21 |
-
import torch
|
| 22 |
-
import yaml
|
| 23 |
-
from einops import rearrange
|
| 24 |
-
from huggingface_hub import hf_hub_download
|
| 25 |
-
|
| 26 |
-
# ==============================================================================
|
| 27 |
-
# --- INITIAL SETUP & CONFIGURATION ---
|
| 28 |
-
# ==============================================================================
|
| 29 |
-
|
| 30 |
-
# Suppress excessive logs from external libraries
|
| 31 |
-
warnings.filterwarnings("ignore")
|
| 32 |
-
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
| 33 |
-
logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s')
|
| 34 |
-
|
| 35 |
-
# --- CONSTANTS ---
|
| 36 |
-
DEPS_DIR = Path("/data")
|
| 37 |
-
LTX_VIDEO_REPO_DIR = DEPS_DIR / "LTX-Video"
|
| 38 |
-
BASE_CONFIG_PATH = LTX_VIDEO_REPO_DIR / "configs"
|
| 39 |
-
DEFAULT_CONFIG_FILE = BASE_CONFIG_PATH / "ltxv-13b-0.9.8-distilled-fp8.yaml"
|
| 40 |
-
LTX_REPO_ID = "Lightricks/LTX-Video"
|
| 41 |
-
RESULTS_DIR = Path("/app/output")
|
| 42 |
-
DEFAULT_FPS = 24.0
|
| 43 |
-
FRAMES_ALIGNMENT = 8
|
| 44 |
-
|
| 45 |
-
# --- DEPENDENCY PATH SETUP ---
|
| 46 |
-
# Ensures the LTX-Video library can be imported
|
| 47 |
-
def add_deps_to_path():
|
| 48 |
-
"""Adds the LTX repository directory to the Python system path."""
|
| 49 |
-
repo_path = str(LTX_VIDEO_REPO_DIR.resolve())
|
| 50 |
-
if repo_path not in sys.path:
|
| 51 |
-
sys.path.insert(0, repo_path)
|
| 52 |
-
logging.info(f"Repo added to sys.path: {repo_path}")
|
| 53 |
-
|
| 54 |
-
add_deps_to_path()
|
| 55 |
-
|
| 56 |
-
# --- PROJECT IMPORTS ---
|
| 57 |
-
# These must come after the path setup
|
| 58 |
-
from api.gpu_manager import gpu_manager
|
| 59 |
-
from ltx_video.models.autoencoders.vae_encode import (normalize_latents, un_normalize_latents)
|
| 60 |
-
from ltx_video.pipelines.pipeline_ltx_video import (ConditioningItem, LTXMultiScalePipeline, adain_filter_latent)
|
| 61 |
-
from ltx_video.pipelines.pipeline_ltx_video import create_ltx_video_pipeline, create_latent_upsampler
|
| 62 |
-
from ltx_video.utils.inference_utils import load_image_to_tensor_with_resize_and_crop
|
| 63 |
-
from managers.vae_manager import vae_manager_singleton
|
| 64 |
-
from tools.video_encode_tool import video_encode_tool_singleton
|
| 65 |
-
|
| 66 |
|
| 67 |
# ==============================================================================
|
| 68 |
-
# ---
|
| 69 |
# ==============================================================================
|
| 70 |
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
torch.manual_seed(seed)
|
| 77 |
-
torch.cuda.manual_seed_all(seed)
|
| 78 |
-
# Potentially faster, but less reproducible
|
| 79 |
-
# torch.backends.cudnn.deterministic = False
|
| 80 |
-
# torch.backends.cudnn.benchmark = True
|
| 81 |
-
|
| 82 |
-
def calculate_padding(orig_h: int, orig_w: int, target_h: int, target_w: int) -> Tuple[int, int, int, int]:
|
| 83 |
-
"""Calculates symmetric padding values to reach a target dimension."""
|
| 84 |
-
pad_h = target_h - orig_h
|
| 85 |
-
pad_w = target_w - orig_w
|
| 86 |
-
pad_top = pad_h // 2
|
| 87 |
-
pad_bottom = pad_h - pad_top
|
| 88 |
-
pad_left = pad_w // 2
|
| 89 |
-
pad_right = pad_w - pad_left
|
| 90 |
-
return (pad_left, pad_right, pad_top, pad_bottom)
|
| 91 |
-
|
| 92 |
-
def log_tensor_info(tensor: torch.Tensor, name: str = "Tensor"):
|
| 93 |
-
"""Logs detailed information about a PyTorch tensor for debugging."""
|
| 94 |
-
if not isinstance(tensor, torch.Tensor):
|
| 95 |
-
logging.debug(f"'{name}' is not a tensor.")
|
| 96 |
-
return
|
| 97 |
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
f"Mean: {tensor.mean().item():.4f}\n"
|
| 110 |
-
)
|
| 111 |
-
except Exception:
|
| 112 |
-
pass # Fails on some dtypes
|
| 113 |
-
logging.debug(info_str + "----------------------")
|
| 114 |
-
|
| 115 |
|
| 116 |
# ==============================================================================
|
| 117 |
-
# ---
|
|
|
|
|
|
|
| 118 |
# ==============================================================================
|
| 119 |
|
| 120 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
"""
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
video generation (low-resolution, upscale).
|
| 125 |
"""
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
"""Initializes the service, loads models, and configures the environment."""
|
| 129 |
-
t0 = time.perf_counter()
|
| 130 |
-
logging.info("Initializing VideoService...")
|
| 131 |
-
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 132 |
-
|
| 133 |
-
self.config = self._load_config(DEFAULT_CONFIG_FILE)
|
| 134 |
-
self._tmp_dirs = set()
|
| 135 |
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
|
|
|
|
|
|
|
|
|
| 141 |
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
)
|
| 148 |
|
| 149 |
-
|
|
|
|
| 150 |
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
# ==========================================================================
|
| 154 |
-
|
| 155 |
-
def _load_config(self, config_path: Path) -> Dict:
|
| 156 |
-
"""Loads the YAML configuration file."""
|
| 157 |
-
logging.info(f"Loading config from: {config_path}")
|
| 158 |
-
with open(config_path, "r") as file:
|
| 159 |
-
return yaml.safe_load(file)
|
| 160 |
-
|
| 161 |
-
def _load_models_on_cpu(self) -> Tuple[LTXMultiScalePipeline, Optional[torch.nn.Module]]:
|
| 162 |
-
"""Downloads and loads the pipeline and upsampler checkpoints onto the CPU."""
|
| 163 |
-
t0 = time.perf_counter()
|
| 164 |
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
repo_id=LTX_REPO_ID,
|
| 168 |
-
filename=self.config["checkpoint_path"],
|
| 169 |
-
token=os.getenv("HF_TOKEN"),
|
| 170 |
-
)
|
| 171 |
-
self.config["checkpoint_path"] = distilled_model_path
|
| 172 |
-
|
| 173 |
-
pipeline = create_ltx_video_pipeline(
|
| 174 |
-
ckpt_path=self.config["checkpoint_path"],
|
| 175 |
-
precision=self.config["precision"],
|
| 176 |
-
device="cpu", # Load on CPU first
|
| 177 |
-
# Pass other config values directly
|
| 178 |
-
**{k: v for k, v in self.config.items() if k in create_ltx_video_pipeline.__code__.co_varnames}
|
| 179 |
-
)
|
| 180 |
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
"
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
def finalize(self, keep_paths: Optional[List[str]] = None):
|
| 214 |
-
"""Cleans up GPU memory and temporary directories."""
|
| 215 |
-
logging.debug("Finalizing resources...")
|
| 216 |
-
gc.collect()
|
| 217 |
-
if torch.cuda.is_available():
|
| 218 |
-
torch.cuda.empty_cache()
|
| 219 |
-
try:
|
| 220 |
-
torch.cuda.ipc_collect()
|
| 221 |
-
except Exception:
|
| 222 |
-
pass
|
| 223 |
|
| 224 |
-
|
| 225 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
-
# ==========================================================================
|
| 228 |
-
# --- PUBLIC ORCHESTRATORS ---
|
| 229 |
-
# These are the main entry points called by the frontend.
|
| 230 |
-
# ==========================================================================
|
| 231 |
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
Returns:
|
| 237 |
-
A tuple of (video_path, latents_path, used_seed).
|
| 238 |
-
"""
|
| 239 |
-
logging.info("Starting narrative low-res generation...")
|
| 240 |
-
used_seed = self._resolve_seed(kwargs.get("seed"))
|
| 241 |
-
seed_everything(used_seed)
|
| 242 |
-
|
| 243 |
-
prompt_list = [p.strip() for p in prompt.splitlines() if p.strip()]
|
| 244 |
-
if not prompt_list:
|
| 245 |
-
raise ValueError("Prompt is empty or contains no valid lines.")
|
| 246 |
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
|
|
|
|
|
|
| 251 |
|
| 252 |
-
|
| 253 |
-
overlap_condition_item = None
|
| 254 |
|
| 255 |
-
|
| 256 |
-
for i, chunk_prompt in enumerate(prompt_list):
|
| 257 |
-
logging.info(f"Generating narrative chunk {i+1}/{num_chunks}: '{chunk_prompt[:50]}...'")
|
| 258 |
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
# Use initial image conditions only for the first chunk
|
| 264 |
-
current_conditions = kwargs.get("initial_conditions", []) if i == 0 else []
|
| 265 |
-
if overlap_condition_item:
|
| 266 |
-
current_conditions.append(overlap_condition_item)
|
| 267 |
-
|
| 268 |
-
chunk_latents = self._generate_single_chunk_low(
|
| 269 |
-
prompt=chunk_prompt,
|
| 270 |
-
num_frames=current_frames,
|
| 271 |
-
seed=used_seed + i,
|
| 272 |
-
conditioning_items=current_conditions,
|
| 273 |
-
**kwargs
|
| 274 |
-
)
|
| 275 |
-
|
| 276 |
-
if chunk_latents is None:
|
| 277 |
-
raise RuntimeError(f"Failed to generate latents for chunk {i+1}.")
|
| 278 |
-
|
| 279 |
-
# Create overlap for the next chunk
|
| 280 |
-
if i < num_chunks - 1:
|
| 281 |
-
overlap_latents = chunk_latents[:, :, -overlap_frames:, :, :].clone()
|
| 282 |
-
log_tensor_info(overlap_latents, f"Overlap Latents from chunk {i+1}")
|
| 283 |
-
overlap_condition_item = ConditioningItem(
|
| 284 |
-
media_item=overlap_latents, media_frame_number=0, conditioning_strength=1.0
|
| 285 |
-
)
|
| 286 |
-
|
| 287 |
-
# Trim the overlap from the current chunk before saving
|
| 288 |
-
if i > 0:
|
| 289 |
-
chunk_latents = chunk_latents[:, :, overlap_frames:, :, :]
|
| 290 |
-
|
| 291 |
-
# Save chunk latents to disk to manage memory
|
| 292 |
-
chunk_path = RESULTS_DIR / f"chunk_{i}_{used_seed}.pt"
|
| 293 |
-
torch.save(chunk_latents.cpu(), chunk_path)
|
| 294 |
-
all_latents_paths.append(chunk_path)
|
| 295 |
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
traceback.print_exc()
|
| 302 |
-
return None, None, None
|
| 303 |
-
finally:
|
| 304 |
-
# Clean up intermediate chunk files
|
| 305 |
-
for path in all_latents_paths:
|
| 306 |
-
if os.path.exists(path):
|
| 307 |
-
os.remove(path)
|
| 308 |
-
self.finalize()
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
def generate_single_low(self, **kwargs) -> Tuple[Optional[str], Optional[str], Optional[int]]:
|
| 312 |
-
"""
|
| 313 |
-
[ORCHESTRATOR] Generates a video from a single prompt in one go.
|
| 314 |
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
"""
|
| 318 |
-
logging.info("Starting single-prompt low-res generation...")
|
| 319 |
-
used_seed = self._resolve_seed(kwargs.get("seed"))
|
| 320 |
-
seed_everything(used_seed)
|
| 321 |
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
final_latents = self._generate_single_chunk_low(
|
| 326 |
-
num_frames=total_frames,
|
| 327 |
-
seed=used_seed,
|
| 328 |
-
conditioning_items=kwargs.get("initial_conditions", []),
|
| 329 |
-
**kwargs
|
| 330 |
-
)
|
| 331 |
-
|
| 332 |
-
if final_latents is None:
|
| 333 |
-
raise RuntimeError("Failed to generate latents.")
|
| 334 |
|
| 335 |
-
|
| 336 |
-
latents_path = RESULTS_DIR / f"single_{used_seed}.pt"
|
| 337 |
-
torch.save(final_latents.cpu(), latents_path)
|
| 338 |
-
return self._finalize_generation([latents_path], "single_video", used_seed)
|
| 339 |
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
return None, None, None
|
| 344 |
-
finally:
|
| 345 |
-
self.finalize()
|
| 346 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
def _generate_single_chunk_low(
|
| 353 |
-
self, prompt: str, negative_prompt: str, height: int, width: int, num_frames: int, seed: int,
|
| 354 |
-
conditioning_items: List[ConditioningItem], ltx_configs_override: Optional[Dict], **kwargs
|
| 355 |
-
) -> Optional[torch.Tensor]:
|
| 356 |
-
"""
|
| 357 |
-
[WORKER] Generates a single chunk of latents. This is the core generation unit.
|
| 358 |
-
Returns the raw latents tensor on the target device, or None on failure.
|
| 359 |
-
"""
|
| 360 |
-
height_padded, width_padded = (self._align(d) for d in (height, width))
|
| 361 |
-
downscale_factor = self.config.get("downscale_factor", 0.6666666)
|
| 362 |
-
vae_scale_factor = self.pipeline.vae_scale_factor
|
| 363 |
-
|
| 364 |
-
downscaled_height = self._align(int(height_padded * downscale_factor), vae_scale_factor)
|
| 365 |
-
downscaled_width = self._align(int(width_padded * downscale_factor), vae_scale_factor)
|
| 366 |
-
|
| 367 |
-
first_pass_config = self.config.get("first_pass", {}).copy()
|
| 368 |
-
if ltx_configs_override:
|
| 369 |
-
first_pass_config.update(self._prepare_guidance_overrides(ltx_configs_override))
|
| 370 |
-
|
| 371 |
-
pipeline_kwargs = {
|
| 372 |
-
"prompt": prompt,
|
| 373 |
-
"negative_prompt": negative_prompt,
|
| 374 |
-
"height": downscaled_height,
|
| 375 |
-
"width": downscaled_width,
|
| 376 |
-
"num_frames": num_frames,
|
| 377 |
-
"frame_rate": DEFAULT_FPS,
|
| 378 |
-
"generator": torch.Generator(device=self.device).manual_seed(seed),
|
| 379 |
-
"output_type": "latent",
|
| 380 |
-
"conditioning_items": conditioning_items,
|
| 381 |
-
**first_pass_config
|
| 382 |
-
}
|
| 383 |
-
|
| 384 |
-
logging.debug(f"Pipeline call args: { {k: v for k, v in pipeline_kwargs.items() if k != 'conditioning_items'} }")
|
| 385 |
|
| 386 |
-
with
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
|
| 392 |
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 396 |
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
# Save final combined latents
|
| 407 |
-
final_latents_path = RESULTS_DIR / f"latents_{base_filename}_{seed}.pt"
|
| 408 |
-
torch.save(final_latents_cpu, final_latents_path)
|
| 409 |
-
logging.info(f"Final latents saved to: {final_latents_path}")
|
| 410 |
-
|
| 411 |
-
# Move to GPU for decoding
|
| 412 |
-
final_latents_gpu = final_latents_cpu.to(self.device)
|
| 413 |
-
log_tensor_info(final_latents_gpu, "Final Concatenated Latents")
|
| 414 |
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 423 |
|
| 424 |
-
def prepare_condition_items(self, items_list: List, height: int, width: int, num_frames: int) -> List[ConditioningItem]:
|
| 425 |
-
"""Prepares a list of ConditioningItem objects from file paths or tensors."""
|
| 426 |
-
if not items_list:
|
| 427 |
-
return []
|
| 428 |
-
|
| 429 |
-
height_padded, width_padded = self._align(height), self._align(width)
|
| 430 |
-
padding_values = calculate_padding(height, width, height_padded, width_padded)
|
| 431 |
-
|
| 432 |
-
conditioning_items = []
|
| 433 |
-
for media, frame, weight in items_list:
|
| 434 |
-
tensor = self._prepare_conditioning_tensor(media, height, width, padding_values)
|
| 435 |
-
safe_frame = max(0, min(int(frame), num_frames - 1))
|
| 436 |
-
conditioning_items.append(ConditioningItem(tensor, safe_frame, float(weight)))
|
| 437 |
-
return conditioning_items
|
| 438 |
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
tensor = torch.nn.functional.pad(tensor, padding)
|
| 443 |
-
log_tensor_info(tensor, f"Prepared Conditioning Tensor from {media_path}")
|
| 444 |
-
return tensor.to(self.device, dtype=self.runtime_autocast_dtype)
|
| 445 |
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
overrides["guidance_scale"] = [1, 1, 4, 5, 4, 1, 1]
|
| 457 |
-
overrides["stg_scale"] = [0, 0, 2, 2, 2, 1, 0]
|
| 458 |
-
elif preset == "Customizado":
|
| 459 |
-
try:
|
| 460 |
-
overrides["guidance_scale"] = json.loads(ltx_configs["guidance_scale_list"])
|
| 461 |
-
overrides["stg_scale"] = json.loads(ltx_configs["stg_scale_list"])
|
| 462 |
-
except (json.JSONDecodeError, KeyError
|
|
|
|
| 1 |
+
# FILE: app_complete.py
|
| 2 |
+
# DESCRIPTION: Gradio web interface for the LTX-Video generation service.
|
| 3 |
+
# Provides a user-friendly, step-by-step workflow for creating videos.
|
| 4 |
|
| 5 |
+
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
import traceback
|
| 7 |
+
import sys
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
# ==============================================================================
|
| 10 |
+
# --- BACKEND SERVICES IMPORT ---
|
| 11 |
# ==============================================================================
|
| 12 |
|
| 13 |
+
# Encapsulate imports in a try-except block for robust error handling at startup.
|
| 14 |
+
try:
|
| 15 |
+
# This assumes the backend file is named 'ltx_server_refactored_complete.py'
|
| 16 |
+
# and is in a reachable path (e.g., 'api/').
|
| 17 |
+
from api.ltx_server_refactored_complete import video_generation_service
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
# Placeholder for SeedVR server.
|
| 20 |
+
# from api.seedvr_server import SeedVRServer
|
| 21 |
+
# seedvr_inference_server = SeedVRServer()
|
| 22 |
+
seedvr_inference_server = None
|
| 23 |
+
print("Backend services imported successfully.")
|
| 24 |
+
except ImportError as e:
|
| 25 |
+
print(f"FATAL ERROR: Could not import backend services. Ensure the backend file is accessible. Details: {e}")
|
| 26 |
+
sys.exit(1)
|
| 27 |
+
except Exception as e:
|
| 28 |
+
print(f"FATAL ERROR: An unexpected error occurred during backend initialization. Details: {e}")
|
| 29 |
+
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
# ==============================================================================
|
| 32 |
+
# --- UI WRAPPER FUNCTIONS ---
|
| 33 |
+
# These functions act as a bridge between the Gradio UI and the backend service.
|
| 34 |
+
# They handle data conversion, error catching, and UI updates.
|
| 35 |
# ==============================================================================
|
| 36 |
|
| 37 |
+
def run_generate_base_video(
|
| 38 |
+
generation_mode: str, prompt: str, neg_prompt: str, start_img: str,
|
| 39 |
+
height: int, width: int, duration: float, seed: int, randomize_seed: bool,
|
| 40 |
+
fp_guidance_preset: str, fp_guidance_scale_list: str, fp_stg_scale_list: str,
|
| 41 |
+
progress=gr.Progress(track_tqdm=True)
|
| 42 |
+
) -> tuple:
|
| 43 |
"""
|
| 44 |
+
Wrapper to call the backend for generating the initial low-resolution video.
|
| 45 |
+
It decides whether to use the 'narrative' or 'single' generation mode.
|
|
|
|
| 46 |
"""
|
| 47 |
+
try:
|
| 48 |
+
print(f"[UI] Request received for base video generation. Mode: {generation_mode}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
+
initial_conditions = []
|
| 51 |
+
if start_img:
|
| 52 |
+
# Estimate total frames for conditioning context
|
| 53 |
+
num_frames_estimate = int(duration * 24)
|
| 54 |
+
items_list = [[start_img, 0, 1.0]] # [[media, frame, weight]]
|
| 55 |
+
initial_conditions = video_generation_service.prepare_condition_items(
|
| 56 |
+
items_list, height, width, num_frames_estimate
|
| 57 |
+
)
|
| 58 |
|
| 59 |
+
# Package advanced LTX settings for the backend
|
| 60 |
+
ltx_configs = {
|
| 61 |
+
"guidance_preset": fp_guidance_preset,
|
| 62 |
+
"guidance_scale_list": fp_guidance_scale_list,
|
| 63 |
+
"stg_scale_list": fp_stg_scale_list,
|
| 64 |
+
}
|
| 65 |
|
| 66 |
+
# Select the appropriate backend function based on UI mode
|
| 67 |
+
if generation_mode == "Narrativa (Múltiplos Prompts)":
|
| 68 |
+
func_to_call = video_generation_service.generate_narrative_low
|
| 69 |
+
else:
|
| 70 |
+
func_to_call = video_generation_service.generate_single_low
|
| 71 |
+
|
| 72 |
+
video_path, tensor_path, final_seed = func_to_call(
|
| 73 |
+
prompt=prompt, negative_prompt=neg_prompt,
|
| 74 |
+
height=height, width=width, duration=duration,
|
| 75 |
+
seed=None if randomize_seed else int(seed),
|
| 76 |
+
initial_conditions=initial_conditions,
|
| 77 |
+
ltx_configs_override=ltx_configs
|
| 78 |
)
|
| 79 |
|
| 80 |
+
if not video_path:
|
| 81 |
+
raise RuntimeError("Backend failed to return a valid video path.")
|
| 82 |
|
| 83 |
+
# Update the session state with the results
|
| 84 |
+
new_state = {"low_res_video": video_path, "low_res_latents": tensor_path, "used_seed": final_seed}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
+
print(f"[UI] Base video generation successful. Path: {video_path}")
|
| 87 |
+
return video_path, new_state, gr.update(visible=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
+
except Exception as e:
|
| 90 |
+
error_message = f"❌ An error occurred during base generation:\n{e}"
|
| 91 |
+
print(f"{error_message}\nDetails: {traceback.format_exc()}")
|
| 92 |
+
raise gr.Error(error_message)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def run_ltx_refinement(
|
| 96 |
+
state: dict, prompt: str, neg_prompt: str,
|
| 97 |
+
progress=gr.Progress(track_tqdm=True)
|
| 98 |
+
) -> tuple:
|
| 99 |
+
"""Wrapper to call the LTX texture refinement and upscaling backend function."""
|
| 100 |
+
if not state or not state.get("low_res_latents"):
|
| 101 |
+
raise gr.Error("Error: Please generate a base video in Step 1 before refining.")
|
| 102 |
+
|
| 103 |
+
try:
|
| 104 |
+
print("[UI] Request received for LTX refinement.")
|
| 105 |
+
video_path, tensor_path = video_generation_service.generate_upscale_denoise(
|
| 106 |
+
latents_path=state["low_res_latents"],
|
| 107 |
+
prompt=prompt,
|
| 108 |
+
negative_prompt=neg_prompt,
|
| 109 |
+
seed=state["used_seed"]
|
| 110 |
+
)
|
| 111 |
+
# Update state with refined assets
|
| 112 |
+
state["refined_video_ltx"] = video_path
|
| 113 |
+
state["refined_latents_ltx"] = tensor_path
|
| 114 |
+
print(f"[UI] LTX refinement successful. Path: {video_path}")
|
| 115 |
+
return video_path, state
|
| 116 |
+
except Exception as e:
|
| 117 |
+
error_message = f"❌ An error occurred during LTX Refinement:\n{e}"
|
| 118 |
+
print(f"{error_message}\nDetails: {traceback.format_exc()}")
|
| 119 |
+
raise gr.Error(error_message)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def run_seedvr_upscaling(
|
| 123 |
+
state: dict, seed: int, resolution: int, batch_size: int, fps: int,
|
| 124 |
+
progress=gr.Progress(track_tqdm=True)
|
| 125 |
+
) -> tuple:
|
| 126 |
+
"""Wrapper to call the SeedVR upscaling backend service."""
|
| 127 |
+
if not state or not state.get("low_res_video"):
|
| 128 |
+
raise gr.Error("Error: Please generate a base video in Step 1 before upscaling.")
|
| 129 |
+
if not seedvr_inference_server:
|
| 130 |
+
raise gr.Error("Error: The SeedVR upscaling server is not available.")
|
| 131 |
+
|
| 132 |
+
try:
|
| 133 |
+
print("[UI] Request received for SeedVR upscaling.")
|
| 134 |
+
def progress_wrapper(p, desc=""): progress(p, desc=desc)
|
| 135 |
|
| 136 |
+
output_filepath = seedvr_inference_server.run_inference(
|
| 137 |
+
file_path=state["low_res_video"], seed=seed, resolution=resolution,
|
| 138 |
+
batch_size=batch_size, fps=fps, progress=progress_wrapper
|
| 139 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
+
status_message = f"✅ Upscaling complete!\nSaved to: {output_filepath}"
|
| 142 |
+
print(f"[UI] SeedVR upscaling successful. Path: {output_filepath}")
|
| 143 |
+
return gr.update(value=output_filepath), gr.update(value=status_message)
|
| 144 |
+
except Exception as e:
|
| 145 |
+
error_message = f"❌ An error occurred during SeedVR Upscaling:\n{e}"
|
| 146 |
+
print(f"{error_message}\nDetails: {traceback.format_exc()}")
|
| 147 |
+
return None, gr.update(value=error_message)
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
|
| 150 |
+
# ==============================================================================
|
| 151 |
+
# --- UI BUILDER ---
|
| 152 |
+
# Functions dedicated to creating parts of the Gradio interface.
|
| 153 |
+
# ==============================================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
|
| 155 |
+
def build_ui():
|
| 156 |
+
"""Constructs the entire Gradio application UI."""
|
| 157 |
+
|
| 158 |
+
with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue")) as demo:
|
| 159 |
+
# App state persists across interactions within a session
|
| 160 |
+
app_state = gr.State(value={"low_res_video": None, "low_res_latents": None, "used_seed": None})
|
| 161 |
|
| 162 |
+
gr.Markdown("# LTX Video - Geração e Pós-Produção por Etapas", elem_id="main-title")
|
|
|
|
| 163 |
|
| 164 |
+
ui_components = {} # Dictionary to hold all key UI components
|
|
|
|
|
|
|
| 165 |
|
| 166 |
+
with gr.Row():
|
| 167 |
+
with gr.Column(scale=1):
|
| 168 |
+
# Build the main generation controls (Step 1)
|
| 169 |
+
_build_generation_controls(ui_components)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
|
| 171 |
+
with gr.Column(scale=1):
|
| 172 |
+
gr.Markdown("### Vídeo Base Gerado")
|
| 173 |
+
ui_components['low_res_video_output'] = gr.Video(
|
| 174 |
+
label="O resultado da Etapa 1 aparecerá aqui", interactive=False
|
| 175 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
|
| 177 |
+
# Build the post-production section (Step 2), initially hidden
|
| 178 |
+
_build_postprod_controls(ui_components)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
|
| 180 |
+
# Connect all UI events to their corresponding functions
|
| 181 |
+
_register_event_handlers(app_state, ui_components)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
+
return demo
|
|
|
|
|
|
|
|
|
|
| 184 |
|
| 185 |
+
def _build_generation_controls(ui: dict):
|
| 186 |
+
"""Builds the UI components for Step 1: Base Video Generation."""
|
| 187 |
+
gr.Markdown("### Etapa 1: Configurações de Geração")
|
|
|
|
|
|
|
|
|
|
| 188 |
|
| 189 |
+
ui['generation_mode'] = gr.Radio(
|
| 190 |
+
label="Modo de Geração",
|
| 191 |
+
choices=["Simples (Prompt Único)", "Narrativa (Múltiplos Prompts)"],
|
| 192 |
+
value="Narrativa (Múltiplos Prompts)",
|
| 193 |
+
info="Simples para uma ação contínua, Narrativa para uma sequência de cenas (uma por linha)."
|
| 194 |
+
)
|
| 195 |
+
ui['prompt'] = gr.Textbox(label="Prompt(s)", value="Um leão majestoso caminha pela savana\nEle sobe em uma grande pedra e olha o horizonte", lines=4)
|
| 196 |
+
ui['neg_prompt'] = gr.Textbox(label="Negative Prompt", value="blurry, low quality, bad anatomy, deformed", lines=2)
|
| 197 |
+
ui['start_image'] = gr.Image(label="Imagem de Início (Opcional)", type="filepath", sources=["upload"])
|
| 198 |
+
|
| 199 |
+
with gr.Accordion("Parâmetros Principais", open=True):
|
| 200 |
+
ui['duration'] = gr.Slider(label="Duração Total (s)", value=4, step=1, minimum=1, maximum=30)
|
| 201 |
+
with gr.Row():
|
| 202 |
+
ui['height'] = gr.Slider(label="Height", value=432, step=16, minimum=256, maximum=1024)
|
| 203 |
+
ui['width'] = gr.Slider(label="Width", value=768, step=16, minimum=256, maximum=1024)
|
| 204 |
+
with gr.Row():
|
| 205 |
+
ui['seed'] = gr.Number(label="Seed", value=42, precision=0)
|
| 206 |
+
ui['randomize_seed'] = gr.Checkbox(label="Randomize Seed", value=True)
|
| 207 |
+
|
| 208 |
+
with gr.Accordion("Opções Avançadas de Guiagem (First Pass)", open=False):
|
| 209 |
+
ui['fp_guidance_preset'] = gr.Dropdown(
|
| 210 |
+
label="Preset de Guiagem",
|
| 211 |
+
choices=["Padrão (Recomendado)", "Agressivo", "Suave", "Customizado"],
|
| 212 |
+
value="Padrão (Recomendado)",
|
| 213 |
+
info="Controla como a guiagem de texto se comporta ao longo da difusão."
|
| 214 |
+
)
|
| 215 |
+
with gr.Group(visible=False) as ui['custom_guidance_group']:
|
| 216 |
+
gr.Markdown("⚠️ Edite as listas em formato JSON. Ex: `[1.0, 2.5, 3.0]`")
|
| 217 |
+
ui['fp_guidance_scale_list'] = gr.Textbox(label="Lista de Guidance Scale", value="[1, 1, 6, 8, 6, 1, 1]")
|
| 218 |
+
ui['fp_stg_scale_list'] = gr.Textbox(label="Lista de STG Scale (Movimento)", value="[0, 0, 4, 4, 4, 2, 1]")
|
| 219 |
+
|
| 220 |
+
ui['generate_low_btn'] = gr.Button("1. Gerar Vídeo Base", variant="primary")
|
| 221 |
|
| 222 |
+
def _build_postprod_controls(ui: dict):
|
| 223 |
+
"""Builds the UI components for Step 2: Post-Production."""
|
| 224 |
+
with gr.Group(visible=False) as ui['post_prod_group']:
|
| 225 |
+
gr.Markdown("--- \n## Etapa 2: Pós-Produção", elem_id="postprod-title")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
+
with gr.Tabs():
|
| 228 |
+
with gr.TabItem("🚀 Upscaler de Textura (LTX)"):
|
| 229 |
+
with gr.Row():
|
| 230 |
+
with gr.Column(scale=1):
|
| 231 |
+
gr.Markdown("Usa o prompt e a semente originais para refinar o vídeo, adicionando detalhes e texturas de alta qualidade.")
|
| 232 |
+
ui['ltx_refine_btn'] = gr.Button("2. Aplicar Refinamento LTX", variant="primary")
|
| 233 |
+
with gr.Column(scale=1):
|
| 234 |
+
ui['ltx_refined_video_output'] = gr.Video(label="Vídeo com Textura Refinada", interactive=False)
|
| 235 |
+
|
| 236 |
+
with gr.TabItem("✨ Upscaler de Resolução (SeedVR)"):
|
| 237 |
+
is_seedvr_available = seedvr_inference_server is not None
|
| 238 |
+
if not is_seedvr_available:
|
| 239 |
+
gr.Markdown("🔴 *O serviço SeedVR não está disponível nesta instância.*")
|
| 240 |
+
|
| 241 |
+
with gr.Row():
|
| 242 |
+
with gr.Column(scale=1):
|
| 243 |
+
ui['seedvr_seed'] = gr.Slider(minimum=0, maximum=999999, value=42, step=1, label="Seed")
|
| 244 |
+
ui['seedvr_resolution'] = gr.Slider(minimum=720, maximum=1440, value=1072, step=8, label="Resolução Vertical")
|
| 245 |
+
ui['seedvr_batch_size'] = gr.Slider(minimum=1, maximum=16, value=4, step=1, label="Batch Size por GPU")
|
| 246 |
+
ui['seedvr_fps'] = gr.Number(label="FPS de Saída (0 = original)", value=0)
|
| 247 |
+
ui['run_seedvr_btn'] = gr.Button("2. Iniciar Upscaling SeedVR", variant="primary", interactive=is_seedvr_available)
|
| 248 |
+
with gr.Column(scale=1):
|
| 249 |
+
ui['seedvr_video_output'] = gr.Video(label="Vídeo com Upscale SeedVR", interactive=False)
|
| 250 |
+
ui['seedvr_status_box'] = gr.Textbox(label="Status", value="Aguardando...", lines=3, interactive=False)
|
| 251 |
|
| 252 |
|
| 253 |
+
# ==============================================================================
|
| 254 |
+
# --- EVENT HANDLERS ---
|
| 255 |
+
# Connects UI component events (like clicks) to the wrapper functions.
|
| 256 |
+
# ==============================================================================
|
| 257 |
+
|
| 258 |
+
def _register_event_handlers(app_state: gr.State, ui: dict):
|
| 259 |
+
"""Registers all Gradio event handlers."""
|
| 260 |
|
| 261 |
+
# --- Handler for custom guidance visibility ---
|
| 262 |
+
def toggle_custom_guidance(preset_choice: str) -> gr.update:
|
| 263 |
+
return gr.update(visible=(preset_choice == "Customizado"))
|
| 264 |
+
|
| 265 |
+
ui['fp_guidance_preset'].change(
|
| 266 |
+
fn=toggle_custom_guidance,
|
| 267 |
+
inputs=ui['fp_guidance_preset'],
|
| 268 |
+
outputs=ui['custom_guidance_group']
|
| 269 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
|
| 271 |
+
# --- Handler for the main "Generate" button ---
|
| 272 |
+
gen_inputs = [
|
| 273 |
+
ui['generation_mode'], ui['prompt'], ui['neg_prompt'], ui['start_image'],
|
| 274 |
+
ui['height'], ui['width'], ui['duration'], ui['seed'], ui['randomize_seed'],
|
| 275 |
+
ui['fp_guidance_preset'], ui['fp_guidance_scale_list'], ui['fp_stg_scale_list']
|
| 276 |
+
]
|
| 277 |
+
gen_outputs = [
|
| 278 |
+
ui['low_res_video_output'], app_state, ui['post_prod_group']
|
| 279 |
+
]
|
| 280 |
+
ui['generate_low_btn'].click(fn=run_generate_base_video, inputs=gen_inputs, outputs=gen_outputs)
|
| 281 |
+
|
| 282 |
+
# --- Handler for the LTX Refine button ---
|
| 283 |
+
refine_inputs = [app_state, ui['prompt'], ui['neg_prompt']]
|
| 284 |
+
refine_outputs = [ui['ltx_refined_video_output'], app_state]
|
| 285 |
+
ui['ltx_refine_btn'].click(fn=run_ltx_refinement, inputs=refine_inputs, outputs=refine_outputs)
|
| 286 |
+
|
| 287 |
+
# --- Handler for the SeedVR Upscale button ---
|
| 288 |
+
if 'run_seedvr_btn' in ui:
|
| 289 |
+
seedvr_inputs = [app_state, ui['seedvr_seed'], ui['seedvr_resolution'], ui['seedvr_batch_size'], ui['seedvr_fps']]
|
| 290 |
+
seedvr_outputs = [ui['seedvr_video_output'], ui['seedvr_status_box']]
|
| 291 |
+
ui['run_seedvr_btn'].click(fn=run_seedvr_upscaling, inputs=seedvr_inputs, outputs=seedvr_outputs)
|
| 292 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 293 |
|
| 294 |
+
# ==============================================================================
|
| 295 |
+
# --- APPLICATION ENTRY POINT ---
|
| 296 |
+
# ==============================================================================
|
|
|
|
|
|
|
|
|
|
| 297 |
|
| 298 |
+
if __name__ == "__main__":
|
| 299 |
+
print("Building Gradio UI...")
|
| 300 |
+
gradio_app = build_ui()
|
| 301 |
+
print("Launching Gradio app...")
|
| 302 |
+
gradio_app.queue().launch(
|
| 303 |
+
server_name="0.0.0.0",
|
| 304 |
+
server_port=7860,
|
| 305 |
+
debug=True,
|
| 306 |
+
show_error=True
|
| 307 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|