Update demo.

VideoLLaMA2/videollama2/__init__.py

@@ -48,9 +48,19 @@ def mm_infer(image_or_video, instruct, model, tokenizer, modal='video', **kwargs
         modal_token = DEFAULT_IMAGE_TOKEN
     elif modal == 'video':
         modal_token = DEFAULT_VIDEO_TOKEN
+    elif modal == 'text':
+        modal_token = ''
     else:
         raise ValueError(f"Unsupported modal: {modal}")
 
+    # 1. vision preprocess (load & transform image or video).
+    if modal == 'text':
+        tensor = None
+    else:
+        tensor = image_or_video.half().cuda()
+        tensor = [(tensor, modal_token)]
+
+    # 2. text preprocess (tag process & generate prompt).
     if isinstance(instruct, str):
         message = [{'role': 'user', 'content': modal_token + '\n' + instruct}]
     elif isinstance(instruct, list):
@@ -76,11 +86,6 @@ def mm_infer(image_or_video, instruct, model, tokenizer, modal='video', **kwargs
     input_ids = tokenizer_multimodal_token(prompt, tokenizer, modal_token, return_tensors='pt').unsqueeze(0).long().cuda()
     attention_masks = input_ids.ne(tokenizer.pad_token_id).long().cuda()
 
-    # 2. vision preprocess (load & transform image or video).
-    tensor = image_or_video.half().cuda()
-
-    tensor = [(tensor, modal_token)]
-
     # 3. generate response according to visual signals and prompts. 
     keywords = [tokenizer.eos_token]
     stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)

VideoLLaMA2/videollama2/eval/eval_video_oqa_activitynet.py

@@ -5,7 +5,7 @@ import time
 import argparse
 import traceback
 from tqdm import tqdm
-from multiprocessing.pool import Pool
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from openai import AzureOpenAI
 
@@ -71,12 +71,13 @@ def prompt_gpt(question, answer, pred, key, qa_set, output_dir):
         json.dump(result_qa_pair, f)
 
 
-def annotate(prediction_set, caption_files, output_dir, args):
+def annotate(task_arg):
     """
     Evaluates question and answer pairs using GPT-3
     Returns a score for correctness.
     """
-    
+    prediction_set, caption_files, output_dir, args = task_arg
+
     for file in tqdm(caption_files):
         key = file[:-5] # Strip file extension
         qa_set = prediction_set[key]
@@ -86,8 +87,8 @@ def annotate(prediction_set, caption_files, output_dir, args):
         try:
             prompt_gpt(question, answer, pred, key, qa_set, output_dir)
         except Exception as e:
-            traceback.print_exc()
             prompt_gpt(question, answer, pred[:50], key, qa_set, output_dir)
+            traceback.print_exc()
 
     time.sleep(1)
 
@@ -141,39 +142,29 @@ def main(args):
             task_args = [(prediction_set, part, args.output_dir, args) for part in all_parts]
 
             # Use a pool of workers to process the files in parallel.
-            with Pool() as pool:
-                pool.starmap(annotate, task_args)
+            with ThreadPoolExecutor(max_workers=args.num_tasks) as executor:
+                list(tqdm(executor.map(annotate, task_args), total=len(task_args)))
 
         except Exception as e:
             print(f"Error: {e}")
 
-    # Combine all the processed files into one
-    combined_contents = {}
-    json_path = args.output_json
-
-    # Iterate through json files
-    for file_name in tqdm(os.listdir(output_dir)):
-        if file_name.endswith(".json"):
-            file_path = os.path.join(output_dir, file_name)
-            with open(file_path, "r") as json_file:
-                try:
-                    content = json.load(json_file)
-                except:
-                    print(json_file)
-                    exit(0)
-                combined_contents[file_name[:-5]] = content
-
-    # Write combined content to a json file
-    with open(json_path, "w") as json_file:
-        json.dump(combined_contents, json_file)
-    print("All evaluation completed!")
+    # multiprocessing to combine json files
+    def combine_json(file_name):
+        file_path = os.path.join(output_dir, file_name)
+        with open(file_path, "r") as json_file:
+            content = json.load(json_file)
+            return (file_name[:-5], content)
+
+    files = os.listdir(output_dir)
+    with ThreadPoolExecutor(max_workers=64) as executor:
+        combined_contents = list(tqdm(executor.map(combine_json, files), total=len(files)))
 
     # Calculate average score and accuracy
     score_sum = 0
     count = 0
     yes_count = 0
     no_count = 0
-    for key, result in tqdm(combined_contents.items()):
+    for key, result in tqdm(combined_contents):
         try:
             # Computing score
             count += 1

VideoLLaMA2/videollama2/serve/gradio_web_server_adhoc.py

@@ -1,6 +1,7 @@
-# import spaces
+import spaces
 
 import os
+import re
 
 import torch
 import gradio as gr
@@ -79,7 +80,7 @@ class Chat:
 
         self.model, self.processor, self.tokenizer = model_init(model_path, load_8bit=load_8bit, load_4bit=load_4bit)
 
-    # @spaces.GPU(duration=120)
+    @spaces.GPU(duration=120)
     @torch.inference_mode()
     def generate(self, data: list, message, temperature, top_p, max_output_tokens):
         # TODO: support multiple turns of conversation.
@@ -95,41 +96,62 @@ class Chat:
         return response
 
 
-# @spaces.GPU(duration=120)
+@spaces.GPU(duration=120)
 def generate(image, video, message, chatbot, textbox_in, temperature, top_p, max_output_tokens, dtype=torch.float16):
     data = []
 
-    image = image if image else "none"
-    video = video if video else "none"
-    assert not (os.path.exists(image) and os.path.exists(video))
-
     processor = handler.processor
-    if os.path.exists(image) and not os.path.exists(video):
-        data.append((processor['image'](image).to(handler.model.device, dtype=dtype), '<image>'))
-    if not os.path.exists(image) and os.path.exists(video):
-        data.append((processor['video'](video).to(handler.model.device, dtype=dtype), '<video>'))
-    if os.path.exists(image) and os.path.exists(video):
-        raise NotImplementedError("Not support image and video at the same time")
+    try:
+        if image is not None:
+            data.append((processor['image'](image).to(handler.model.device, dtype=dtype), '<image>'))
+        elif video is not None:
+            data.append((processor['video'](video).to(handler.model.device, dtype=dtype), '<video>'))
+        elif image is None and video is None:
+            data.append((None, '<text>'))
+        else:
+            raise NotImplementedError("Not support image and video at the same time")
+    except Exception as e:
+        traceback.print_exc()
+        return gr.update(value=None, interactive=True), gr.update(value=None, interactive=True), message, chatbot
 
     assert len(message) % 2 == 0, "The message should be a pair of user and system message."
 
-    message.append({'role': 'user', 'content': textbox_in})
-    text_en_out = handler.generate(data, message, temperature=temperature, top_p=top_p, max_output_tokens=max_output_tokens)
-    message.append({'role': 'assistant', 'content': text_en_out})
-
     show_images = ""
-    if os.path.exists(image):
+    if image is not None:
         show_images += f'<img src="./file={image}" style="display: inline-block;width: 250px;max-height: 400px;">'
-    if os.path.exists(video):
+    if video is not None:
         show_images += f'<video controls playsinline width="500" style="display: inline-block;"  src="./file={video}"></video>'
 
-    chatbot.append([textbox_in + "\n" + show_images, text_en_out])
+    one_turn_chat = [textbox_in, None]
+
+    # 1. first run case
+    if len(chatbot) == 0:
+        one_turn_chat[0] += "\n" + show_images
+    # 2. not first run case
+    else:
+        previous_image = re.findall(r'<img src="./file=(.+?)"', chatbot[0][0])
+        previous_video = re.findall(r'<video controls playsinline width="500" style="display: inline-block;"  src="./file=(.+?)"', chatbot[0][0])
+        if len(previous_image) > 0:
+            previous_image = previous_image[0]
+            # 2.1 new image append or pure text input will start a new conversation
+            if previous_image != image:
+                message.clear()
+                one_turn_chat[0] += "\n" + show_images if image is not None else ""
+        elif len(previous_video) > 0:
+            previous_video = previous_video[0]
+            # 2.2 new video append or pure text input will start a new conversation
+            if previous_video != video:
+                message.clear()
+                one_turn_chat[0] += "\n" + show_images if video is not None else ""
+
+    message.append({'role': 'user', 'content': textbox_in})
+    text_en_out = handler.generate(data, message, temperature=temperature, top_p=top_p, max_output_tokens=max_output_tokens)
+    message.append({'role': 'assistant', 'content': text_en_out})
+
+    one_turn_chat[1] = text_en_out
+    chatbot.append(one_turn_chat)
 
-    return (
-        gr.update(value=image if os.path.exists(image) else None, interactive=True),
-        gr.update(value=video if os.path.exists(video) else None, interactive=True), 
-        message,
-        chatbot)
+    return gr.update(value=image, interactive=True), gr.update(value=video, interactive=True), message, chatbot
 
 
 def regenerate(message, chatbot):
@@ -147,7 +169,7 @@ def clear_history(message, chatbot):
 
 
 # BUG of Zero Environment
-# 1. The environment is fixed to torch==2.0.1+cu117, gradio>=4.x.x
+# 1. The environment is fixed to torch>=2.0,<=2.2, gradio>=4.x.x
 # 2. The operation or tensor which requires cuda are limited in those functions wrapped via spaces.GPU
 # 3. The function can't return tensor or other cuda objects.
 

app.py

@@ -1,6 +1,7 @@
 import spaces
 
 import os
+import re
 
 import torch
 import gradio as gr
@@ -19,7 +20,6 @@ title_markdown = ("""
   <div>
     <h1 >VideoLLaMA 2: Advancing Spatial-Temporal Modeling and Audio Understanding in Video-LLMs</h1>
     <h5 style="margin: 0;">If this demo please you, please give us a star ⭐ on Github or 💖 on this space.</h5>
-    <h6 style="margin: 0;">Note that the current demo only supports <b>vision input</b> and <b>single-turn conversation</b>. More features will be available soon.</h6>
   </div>
 </div>
 
@@ -100,37 +100,58 @@ class Chat:
 def generate(image, video, message, chatbot, textbox_in, temperature, top_p, max_output_tokens, dtype=torch.float16):
     data = []
 
-    image = image if image else "none"
-    video = video if video else "none"
-    assert not (os.path.exists(image) and os.path.exists(video))
-
     processor = handler.processor
-    if os.path.exists(image) and not os.path.exists(video):
-        data.append((processor['image'](image).to(handler.model.device, dtype=dtype), '<image>'))
-    if not os.path.exists(image) and os.path.exists(video):
-        data.append((processor['video'](video).to(handler.model.device, dtype=dtype), '<video>'))
-    if os.path.exists(image) and os.path.exists(video):
-        raise NotImplementedError("Not support image and video at the same time")
+    try:
+        if image is not None:
+            data.append((processor['image'](image).to(handler.model.device, dtype=dtype), '<image>'))
+        elif video is not None:
+            data.append((processor['video'](video).to(handler.model.device, dtype=dtype), '<video>'))
+        elif image is None and video is None:
+            data.append((None, '<text>'))
+        else:
+            raise NotImplementedError("Not support image and video at the same time")
+    except Exception as e:
+        traceback.print_exc()
+        return gr.update(value=None, interactive=True), gr.update(value=None, interactive=True), message, chatbot
 
     assert len(message) % 2 == 0, "The message should be a pair of user and system message."
 
-    message.append({'role': 'user', 'content': textbox_in})
-    text_en_out = handler.generate(data, message, temperature=temperature, top_p=top_p, max_output_tokens=max_output_tokens)
-    message.append({'role': 'assistant', 'content': text_en_out})
-
     show_images = ""
-    if os.path.exists(image):
+    if image is not None:
         show_images += f'<img src="./file={image}" style="display: inline-block;width: 250px;max-height: 400px;">'
-    if os.path.exists(video):
+    if video is not None:
         show_images += f'<video controls playsinline width="500" style="display: inline-block;"  src="./file={video}"></video>'
 
-    chatbot.append([textbox_in + "\n" + show_images, text_en_out])
+    one_turn_chat = [textbox_in, None]
+
+    # 1. first run case
+    if len(chatbot) == 0:
+        one_turn_chat[0] += "\n" + show_images
+    # 2. not first run case
+    else:
+        previous_image = re.findall(r'<img src="./file=(.+?)"', chatbot[0][0])
+        previous_video = re.findall(r'<video controls playsinline width="500" style="display: inline-block;"  src="./file=(.+?)"', chatbot[0][0])
+        if len(previous_image) > 0:
+            previous_image = previous_image[0]
+            # 2.1 new image append or pure text input will start a new conversation
+            if previous_image != image:
+                message.clear()
+                one_turn_chat[0] += "\n" + show_images if image is not None else ""
+        elif len(previous_video) > 0:
+            previous_video = previous_video[0]
+            # 2.2 new video append or pure text input will start a new conversation
+            if previous_video != video:
+                message.clear()
+                one_turn_chat[0] += "\n" + show_images if video is not None else ""
+
+    message.append({'role': 'user', 'content': textbox_in})
+    text_en_out = handler.generate(data, message, temperature=temperature, top_p=top_p, max_output_tokens=max_output_tokens)
+    message.append({'role': 'assistant', 'content': text_en_out})
 
-    return (
-        gr.update(value=image if os.path.exists(image) else None, interactive=True),
-        gr.update(value=video if os.path.exists(video) else None, interactive=True), 
-        message,
-        chatbot)
+    one_turn_chat[1] = text_en_out
+    chatbot.append(one_turn_chat)
+
+    return gr.update(value=image, interactive=True), gr.update(value=video, interactive=True), message, chatbot
 
 
 def regenerate(message, chatbot):
@@ -148,26 +169,25 @@ def clear_history(message, chatbot):
 
 
 # BUG of Zero Environment
-# 1. The environment is fixed to torch==2.0.1+cu117, gradio>=4.x.x
+# 1. The environment is fixed to torch>=2.0,<=2.2, gradio>=4.x.x
 # 2. The operation or tensor which requires cuda are limited in those functions wrapped via spaces.GPU
 # 3. The function can't return tensor or other cuda objects.
 
-conv_mode = "llama_2"
 model_path = 'DAMO-NLP-SG/VideoLLaMA2-7B-16F'
 
-device = torch.device("cuda")
-
 handler = Chat(model_path, load_8bit=False, load_4bit=True)
 
 textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False)
 
 theme = gr.themes.Default(primary_hue=plum_color)
+# theme.update_color("primary", plum_color.c500)
 theme.set(slider_color="#9C276A")
 theme.set(block_title_text_color="#9C276A")
 theme.set(block_label_text_color="#9C276A")
 theme.set(button_primary_text_color="#9C276A")
 # theme.set(button_secondary_text_color="*neutral_800")
 
+
 with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as demo:
     gr.Markdown(title_markdown)
     message = gr.State([])
@@ -235,16 +255,16 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
             gr.Examples(
                 examples=[
                     [
-                        f"{cur_dir}/examples/sora.png",
+                        f"{cur_dir}/examples/extreme_ironing.jpg",
                         "What happens in this image?",
                     ],
                     [
                         f"{cur_dir}/examples/waterview.jpg",
-                        "What should I be cautious about when I visit the scenic area in this image?",
+                        "What are the things I should be cautious about when I visit here?",
                     ],
                     [
                         f"{cur_dir}/examples/desert.jpg",
-                        "If there are factual errors in the questions, point them out; if not, proceed to answer the following question. What’s happening in the desert?",
+                        "If there are factual errors in the questions, point it out; if not, proceed answering the question. What’s happening in the desert?",
                     ],
                 ],
                 inputs=[image, textbox],
@@ -253,22 +273,22 @@ with gr.Blocks(title='VideoLLaMA 2 🔥🚀🔥', theme=theme, css=block_css) as
             gr.Examples(
                 examples=[
                     [
-                        f"{cur_dir}/examples/rap.mp4",
+                        f"{cur_dir}/../../assets/cat_and_chicken.mp4",
                         "What happens in this video?",
                     ],
                     [
-                        f"{cur_dir}/examples/demo2.mp4",
-                        "Do you think it's morning or night in this video? Why?",
+                        f"{cur_dir}/../../assets/sora.mp4",
+                        "Please describe this video.",
                     ],
                     [
-                        f"{cur_dir}/examples/demo3.mp4",
-                        "At the intersection, in which direction does the red car turn?",
+                        f"{cur_dir}/examples/sample_demo_1.mp4",
+                        "What does the baby do?",
                     ],
                 ],
                 inputs=[video, textbox],
             )
 
-    # gr.Markdown(tos_markdown)
+    gr.Markdown(tos_markdown)
     gr.Markdown(learn_more_markdown)
 
     submit_btn.click(