v3

app.py

@@ -1,49 +1,65 @@
 import gradio as gr
-from smolagents import CodeAgent, HfApiModel
-from gradio_client import Client
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+import torch
 import tempfile
 
-# Initialize Qwen2.5-Omni-7B via SmolAgents
-qwen_agent = CodeAgent(
-    tools=[],  # Add multimodal tools as needed
-    model=HfApiModel("Qwen/Qwen2.5-Omni-7B"),
-    execution_timeout=120
+# Initialize Qwen2.5-Omni-7B with multimodal support
+model = AutoModelForCausalLM.from_pretrained(
+    "Qwen/Qwen2.5-Omni-7B",
+    torch_dtype=torch.float16,
+    device_map="auto"
 )
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Omni-7B")
 
-def process_video(video_path, prompt, request: gr.Request):
-    # Handle ZeroGPU rate limiting
+def analyze_media(video_path, prompt, request: gr.Request):
+    # ZeroGPU rate limiting headers
     headers = {"X-IP-Token": request.headers.get('x-ip-token', '')}
 
-    # Process video with Qwen's multimodal capabilities
-    response = qwen_agent.run(
-        f"Analyze this video: {video_path} and {prompt}",
-        headers=headers
+    # Create multimodal pipeline
+    pipe = pipeline(
+        "multimodal-generation",
+        model=model,
+        tokenizer=tokenizer,
+        device=model.device,
+        max_new_tokens=1024,
+        generate_speech=True
     )
 
-    # Generate real-time speech response
-    audio_response = qwen_agent.model.generate_speech(response)
+    # Process 120s video with TMRoPE alignment
+    result = pipe(
+        media=video_path,
+        text=prompt,
+        headers=headers,
+        timeout=120
+    )
 
-    return response, audio_response
+    # Save speech output to temporary file
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+        result["speech"].export(f.name, format="wav")
+        return result["text"], f.name
 
 with gr.Blocks() as demo:
-    gr.Markdown("## Multimodal AI Demo with Qwen2.5-Omni-7B")
+    gr.Markdown("## Qwen2.5-Omni-7B Multimodal Demo")
 
     with gr.Row():
-        video_input = gr.Video(label="Upload Video", sources=["upload"])
-        prompt_input = gr.Textbox(label="Analysis Prompt")
+        media_input = gr.Video(
+            label="Upload Video (max 120s)",
+            sources=["upload"],
+            max_length=120
+        )
+        prompt_input = gr.Textbox(label="Analysis Prompt", placeholder="Describe or ask about the video...")
 
-    submit_btn = gr.Button("Analyze")
+    submit_btn = gr.Button("Analyze", variant="primary")
 
     with gr.Column():
-        text_output = gr.Textbox(label="Analysis Results")
-        audio_output = gr.Audio(label="Voice Response", autoplay=True)
+        text_output = gr.Textbox(label="Analysis Results", interactive=False)
+        audio_output = gr.Audio(label="Speech Response", autoplay=True)
 
     submit_btn.click(
-        process_video,
-        inputs=[video_input, prompt_input, gr.Request()],
+        analyze_media,
+        inputs=[media_input, prompt_input, gr.Request()],
         outputs=[text_output, audio_output]
     )
 
-# ZeroGPU configuration
-demo.queue(default_concurrency_limit=5)
+demo.queue(concurrency_count=2)
 demo.launch(server_name="0.0.0.0", server_port=7860)

requirements.txt

@@ -1,3 +1,4 @@
-smolagents>=0.9.0
-gradio_client>=1.8.0
-qwen2.5-omni
+torch>=2.3.0
+transformers>=4.41.0
+gradio>=4.26.0
+soundfile>=0.12.1