#!/usr/bin/env python3 """ ZipVoice Gradio Web Interface for HuggingFace Spaces Updated for Gradio 5.47.0 compatibility """ import os import sys import json import tempfile import gradio as gr import torch from pathlib import Path import spaces import whisper # Add current directory to Python path for local zipvoice package sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # Import ZipVoice components from zipvoice.models.zipvoice import ZipVoice from zipvoice.models.zipvoice_distill import ZipVoiceDistill from zipvoice.tokenizer.tokenizer import EmiliaTokenizer from zipvoice.utils.checkpoint import load_checkpoint from zipvoice.utils.feature import VocosFbank from zipvoice.bin.infer_zipvoice import generate_sentence from lhotse.utils import fix_random_seed # Global caches for lazy loading _models_cache: dict[str, dict[str, object]] = {} _tokenizer_cache: EmiliaTokenizer | None = None _vocoder_cache = None _feature_extractor_cache = None def load_models_and_components(model_name: str): """Load and cache models, tokenizer, vocoder, and feature extractor.""" global _models_cache, _tokenizer_cache, _vocoder_cache, _feature_extractor_cache device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if model_name not in _models_cache: print(f"Loading {model_name} model…") model_dir_map = { "zipvoice": "zipvoice", "zipvoice_distill": "zipvoice_distill", } huggingface_repo = "k2-fsa/ZipVoice" from huggingface_hub import hf_hub_download model_ckpt = hf_hub_download(huggingface_repo, filename=f"{model_dir_map[model_name]}/model.pt") model_config_path = hf_hub_download(huggingface_repo, filename=f"{model_dir_map[model_name]}/model.json") token_file = hf_hub_download(huggingface_repo, filename=f"{model_dir_map[model_name]}/tokens.txt") if _tokenizer_cache is None: _tokenizer_cache = EmiliaTokenizer(token_file=token_file) tokenizer = _tokenizer_cache tokenizer_config = {"vocab_size": tokenizer.vocab_size, "pad_id": tokenizer.pad_id} with open(model_config_path, "r") as f: model_config = json.load(f) if model_name == "zipvoice": model = ZipVoice(**model_config["model"], **tokenizer_config) else: model = ZipVoiceDistill(**model_config["model"], **tokenizer_config) load_checkpoint(filename=model_ckpt, model=model, strict=True) model = model.to(device) model.eval() _models_cache[model_name] = { "model": model, "sampling_rate": model_config["feature"]["sampling_rate"], } if _vocoder_cache is None: from vocos import Vocos _vocoder_cache = Vocos.from_pretrained("charactr/vocos-mel-24khz") _vocoder_cache = _vocoder_cache.to(device) _vocoder_cache.eval() if _feature_extractor_cache is None: _feature_extractor_cache = VocosFbank() entry = _models_cache[model_name] return ( entry["model"], _tokenizer_cache, _vocoder_cache, _feature_extractor_cache, entry["sampling_rate"], ) @spaces.GPU def transcribe_audio_whisper(audio_file): """Transcribe audio file using Whisper.""" if audio_file is None: return "Error: Please upload an audio file first." try: model = whisper.load_model("small") with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_audio: temp_audio_path = temp_audio.name with open(temp_audio_path, "wb") as f: f.write(audio_file) result = model.transcribe(temp_audio_path) os.unlink(temp_audio_path) return result["text"].strip() except Exception as exc: # pylint: disable=broad-except return f"Error during transcription: {exc}" @spaces.GPU def synthesize_speech_gradio( text: str, prompt_audio_file, prompt_text: str, model_name: str, speed: float, ): """Synthesize speech using ZipVoice for Gradio interface.""" if not text.strip(): return None, "Error: Please enter text to synthesize." if prompt_audio_file is None: return None, "Error: Please upload a prompt audio file." if not prompt_text.strip(): return None, "Error: Please enter the transcription of the prompt audio." try: fix_random_seed(666) model, tokenizer, vocoder, feature_extractor, sampling_rate = load_models_and_components(model_name) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_audio: temp_audio_path = temp_audio.name with open(temp_audio_path, "wb") as f: f.write(prompt_audio_file) with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_output: output_path = temp_output.name print(f"Synthesizing: '{text}' using {model_name}") print(f"Prompt: {prompt_text}") print(f"Speed: {speed}") with torch.inference_mode(): metrics = generate_sentence( save_path=output_path, prompt_text=prompt_text, prompt_wav=temp_audio_path, text=text, model=model, vocoder=vocoder, tokenizer=tokenizer, feature_extractor=feature_extractor, device=device, num_step=16 if model_name == "zipvoice" else 8, guidance_scale=1.0 if model_name == "zipvoice" else 3.0, speed=speed, t_shift=0.5, target_rms=0.1, feat_scale=0.1, sampling_rate=sampling_rate, max_duration=100, remove_long_sil=False, ) with open(output_path, "rb") as f: audio_data = f.read() os.unlink(temp_audio_path) os.unlink(output_path) success_msg = f"Synthesis completed! Duration: {metrics['wav_seconds']:.2f}s, RTF: {metrics['rtf']:.2f}" return audio_data, success_msg except Exception as exc: # pylint: disable=broad-except error_msg = f"Error during synthesis: {exc}" print(error_msg) return None, error_msg def create_gradio_interface(): """Create the Gradio web interface.""" gpu_available = torch.cuda.is_available() css = """ :root { --primary-gradient: linear-gradient(135deg, #667eea 0%, #764ba2 100%); --accent-gradient: linear-gradient(135deg, #f093fb 0%, #f5576c 100%); --success-gradient: linear-gradient(135deg, #4facfe 0%, #00f2fe 100%); --warning-gradient: linear-gradient(135deg, #fa709a 0%, #fee140 100%); --surface: #ffffff; --surface-muted: #f8fafc; --surface-soft: #f1f5f9; --text-strong: #0f172a; --text: #1f2937; --text-muted: #64748b; --border: #e2e8f0; --shadow-sm: 0 1px 3px rgba(15, 23, 42, 0.08); --shadow-md: 0 8px 24px rgba(15, 23, 42, 0.08); --radius-sm: 8px; --radius-md: 14px; --radius-lg: 20px; } body { background: var(--surface-muted); } .gradio-container { max-width: 1180px; margin: 0 auto; padding: 0 24px 48px; font-family: "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; color: var(--text-strong); } .header-section { background: var(--surface); border-radius: var(--radius-lg); padding: 2.4rem; margin: 2.5rem 0 2rem; box-shadow: var(--shadow-md); border: 1px solid var(--border); } .logo-section { display: flex; align-items: center; gap: 1rem; } .logo-icon { font-size: 3rem; background: var(--primary-gradient); -webkit-background-clip: text; color: transparent; } .title { font-size: 2.6rem; font-weight: 800; background: var(--primary-gradient); -webkit-background-clip: text; color: transparent; margin: 0; letter-spacing: -0.03em; } .subtitle { margin: 0.35rem 0 0; font-size: 1.05rem; color: var(--text-muted); font-weight: 500; } .status-badge { display: inline-flex; align-items: center; gap: 0.5rem; padding: 0.55rem 1.2rem; border-radius: 999px; font-size: 0.85rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.08em; color: #fff; box-shadow: var(--shadow-sm); } .status-badge.gpu { background: var(--success-gradient); } .status-badge.cpu { background: var(--warning-gradient); } .steps-row { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 1rem; margin-bottom: 2rem; } .step-chip { background: var(--surface); border-radius: var(--radius-md); padding: 1rem 1.2rem; display: flex; flex-direction: column; gap: 0.35rem; box-shadow: var(--shadow-sm); border: 1px solid var(--border); } .step-chip span { font-size: 0.75rem; font-weight: 700; text-transform: uppercase; letter-spacing: 0.12em; color: var(--text-muted); } .step-chip strong { font-size: 0.95rem; color: var(--text-strong); } .layout-grid { display: grid; grid-template-columns: minmax(0, 3fr) minmax(0, 2fr); gap: 2rem; align-items: start; margin-bottom: 2.5rem; } .input-card, .output-card { background: var(--surface); border-radius: var(--radius-lg); padding: 1.8rem; box-shadow: var(--shadow-md); border: 1px solid var(--border); display: flex; flex-direction: column; gap: 1.25rem; } .section-title { font-size: 1.2rem; font-weight: 700; display: flex; align-items: center; gap: 0.6rem; color: var(--text-strong); } .section-subtitle { font-size: 0.95rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.1em; color: var(--text-muted); } .helper-text { font-size: 0.85rem; color: var(--text-muted); margin-top: -0.35rem; } .file-drop { border: 2px dashed var(--border) !important; border-radius: var(--radius-md) !important; background: var(--surface-soft) !important; transition: all 0.25s ease; padding: 1rem; } .file-drop:hover { border-color: #667eea !important; background: rgba(102, 126, 234, 0.08) !important; } .button-row { display: flex; gap: 0.6rem; flex-wrap: wrap; } .btn-primary { background: var(--primary-gradient) !important; color: #fff !important; border: none !important; border-radius: var(--radius-md) !important; font-weight: 600 !important; letter-spacing: 0.05em; padding: 0.9rem 1.6rem !important; box-shadow: var(--shadow-md); transition: transform 0.2s ease, box-shadow 0.2s ease; } .btn-secondary { background: var(--surface-soft) !important; color: var(--text-strong) !important; border-radius: var(--radius-md) !important; border: 1px solid var(--border) !important; font-weight: 600 !important; padding: 0.75rem 1.4rem !important; transition: transform 0.2s ease, box-shadow 0.2s ease; } .btn-danger { background: var(--warning-gradient) !important; color: #fff !important; border-radius: var(--radius-md) !important; border: none !important; font-weight: 600 !important; padding: 0.75rem 1.2rem !important; transition: transform 0.2s ease, box-shadow 0.2s ease; } .btn-primary:hover, .btn-secondary:hover, .btn-danger:hover { transform: translateY(-1px); box-shadow: var(--shadow-md); } .divider { height: 1px; width: 100%; background: var(--border); margin: 0.5rem 0 0.75rem; } .text-area textarea, .text-input textarea, .text-input input { background: var(--surface-soft); border: 1.5px solid var(--border); border-radius: var(--radius-md); transition: border-color 0.25s ease, box-shadow 0.25s ease; font-size: 1rem; } .text-area textarea:focus, .text-input textarea:focus, .text-input input:focus { border-color: #667eea; box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.15); background: var(--surface); } .advanced-settings { border-radius: var(--radius-md); background: var(--surface-soft); border: 1px solid var(--border); box-shadow: var(--shadow-sm); } .status-box { background: var(--surface-soft); border: 1px solid rgba(102, 126, 234, 0.25); border-radius: var(--radius-md); padding: 1rem; font-size: 0.95rem; color: #334155; box-shadow: inset 0 1px 2px rgba(15, 23, 42, 0.05); min-height: 82px; } .status-box pre { white-space: pre-wrap; } .progress-indicator { display: none; } .progress-indicator.active { display: flex; align-items: center; gap: 0.85rem; background: rgba(102, 126, 234, 0.1); border: 1px solid rgba(102, 126, 234, 0.25); border-radius: var(--radius-md); padding: 0.85rem 1.1rem; color: #4c51bf; font-weight: 600; } .progress-indicator .spinner { width: 18px; height: 18px; border-radius: 50%; border: 3px solid rgba(102, 126, 234, 0.25); border-top-color: #6366f1; animation: spin 1s linear infinite; } @keyframes spin { to { transform: rotate(360deg); } } .audio-player { background: var(--surface-soft); border-radius: var(--radius-md); border: 1px solid var(--border); padding: 1rem; } .audio-player button.download { background: var(--primary-gradient) !important; color: #fff !important; border-radius: var(--radius-sm) !important; border: none !important; font-weight: 600 !important; margin-top: 0.75rem; box-shadow: var(--shadow-sm); } .examples-deck { background: var(--surface); border-radius: var(--radius-lg); padding: 1.6rem; box-shadow: var(--shadow-md); border: 1px solid var(--border); } .examples-deck .section-title { margin-bottom: 1rem; } .footer { text-align: center; margin-top: 2.5rem; padding: 1.5rem; background: var(--surface); border-radius: var(--radius-lg); border: 1px solid var(--border); box-shadow: var(--shadow-sm); color: var(--text-muted); font-size: 0.9rem; } .footer-links { margin-top: 0.75rem; display: flex; justify-content: center; gap: 1.75rem; } .footer-link { color: var(--text-muted); text-decoration: none; font-weight: 600; } .footer-link:hover { color: #6366f1; } @media (max-width: 1024px) { .layout-grid { grid-template-columns: 1fr; } } @media (max-width: 768px) { .gradio-container { padding: 0 16px 32px; } .header-section { padding: 1.8rem; } .logo-section { flex-direction: column; text-align: center; gap: 0.6rem; } .title { font-size: 2.1rem; } .steps-row { grid-template-columns: 1fr; } .button-row { flex-direction: column; } } @media (prefers-color-scheme: dark) { :root { --surface: #1f2937; --surface-muted: #0f172a; --surface-soft: #273549; --text-strong: #f8fafc; --text: #e2e8f0; --text-muted: #94a3b8; --border: #324155; } .status-box { border-color: rgba(99, 102, 241, 0.45); color: #cbd5f5; } .progress-indicator.active { background: rgba(99, 102, 241, 0.2); border-color: rgba(99, 102, 241, 0.4); color: #cbd5f5; } } """ with gr.Blocks(title="ZipVoice — Zero-Shot TTS", css=css, theme=gr.themes.Soft()) as interface: with gr.Column(elem_classes="header-section"): with gr.Row(): with gr.Column(scale=3): gr.HTML("""
Zero-shot text-to-speech with instant voice cloning
Tip: use a clear 1–3 second sample for best results. 提示:請使用 1–3 秒的清晰語音,以獲得最佳效果。
") gr.HTML("