Update app.py

app.py

@@ -77,7 +77,7 @@ processor = AutoProcessor.from_pretrained(MODEL_ID_QWEN, trust_remote_code=True)
 qwen_model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
     MODEL_ID_QWEN,
     trust_remote_code=True,
-    torch_dtype=torch.bfloat16
+    torch_dtype=torch.float16  # Use float16 for more stability
 ).to("cuda").eval()
 
 # -----------------------
@@ -136,6 +136,8 @@ def model_inference(input_dict, history):
             return_tensors="pt",
             padding=True,
         ).to("cuda")
+        # Clear CUDA cache to reduce potential memory fragmentation.
+        torch.cuda.empty_cache()
         # Set up streaming generation.
         streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
         generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024)
@@ -155,7 +157,7 @@ def model_inference(input_dict, history):
     if not files:
         # Prepare a simple conversation for text-only input.
         conversation = [{"role": "user", "content": text}]
-        # Here we use the text tokenizer’s chat template method.
+        # Use the text tokenizer’s chat template method.
         input_ids = text_tokenizer.apply_chat_template(
             conversation, add_generation_prompt=True, return_tensors="pt"
         )
@@ -216,6 +218,8 @@ def model_inference(input_dict, history):
         return_tensors="pt",
         padding=True,
     ).to("cuda")
+    # Clear CUDA cache before generation.
+    torch.cuda.empty_cache()
     streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
     generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024)
     thread = Thread(target=qwen_model.generate, kwargs=generation_kwargs)
@@ -235,6 +239,7 @@ examples = [
     [{"text": "Tell me a story about a brave knight in a faraway kingdom."}],
     [{"text": "@video-infer Explain the content of the Advertisement", "files": ["example_images/videoplayback.mp4"]}],
     [{"text": "@video-infer Explain the content of the video in detail", "files": ["example_images/breakfast.mp4"]}],
+    [{"text": "@video-infer Explain the content of the video.", "files": ["example_images/sky.mp4"]}],
 
 ]
 
@@ -250,4 +255,4 @@ demo = gr.ChatInterface(
 )
 
 if __name__ == "__main__":
-    demo.launch(debug=True)
+    demo.launch(share=True, debug=True)