Update app.py

app.py

@@ -16,16 +16,14 @@ import uuid
 
 
 welcome_message = """
-# 👋🏻Welcome to ⚕🗣️😷MultiMed - Access Chat ⚕🗣️😷
+# 👋🏻Welcome to ⚕🗣️😷TruEra - MultiMed ⚕🗣️😷
 
-🗣️📝 This is an educational and accessible conversational tool.
+🗣️📝 This is an accessible and multimodal tool optimized using TruEra! We evaluated several configurations, prompts, and models to optimize this application. 
 
-### How To Use ⚕🗣️😷MultiMed⚕: 
+### How To Use ⚕🗣️😷TruEra - MultiMed⚕: 
 
-🗣️📝Interact with ⚕🗣️😷MultiMed⚕ in any language using image, audio or text!
-
-📚🌟💼 that uses [Tonic/stablemed](https://huggingface.co/Tonic/stablemed) and [adept/fuyu-8B](https://huggingface.co/adept/fuyu-8b) with [Vectara](https://huggingface.co/vectara) embeddings + retrieval w/ [Facebook/Seamless-m4t](https://huggingface.co/facebook/hf-seamless-m4t-large) for audio translation & accessibility.
-do [get in touch](https://discord.gg/GWpVpekp). You can also use 😷MultiMed⚕️ on your own data & in your own way by cloning this space. 🧬🔬🔍 Simply click here: <a style="display:inline-block" href="https://huggingface.co/spaces/TeamTonic/MultiMed?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a></h3>
+🗣️📝Interact with ⚕🗣️😷TruEra - MultiMed⚕ in any language using image, audio or text. ⚕🗣️😷TruEra - MultiMed is an accessible application 📚🌟💼 that uses [Qwen/Qwen-1_8B-Chat](https://huggingface.co/Qwen/Qwen-1_8B-Chat) and [Tonic1/Official-Qwen-VL-Chat](https://huggingface.co/Qwen/Qwen-VL-Chat) with [Vectara](https://huggingface.co/vectara) embeddings + retrieval w/ [facebook/seamless-m4t-v2-large](https://huggingface.co/facebook/hf-seamless-m4t-large) for audio translation & accessibility.
+do [get in touch](https://discord.gg/GWpVpekp). You can also use 😷TruEra MultiMed⚕️ on your own data & in your own way by cloning this space. 🧬🔬🔍 Simply click here: <a style="display:inline-block" href="https://huggingface.co/spaces/TeamTonic/MultiMed?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a></h3>
 ### Join us : 
 
 🌟TeamTonic🌟 is always making cool demos! Join our active builder's🛠️community on 👻Discord: [Discord](https://discord.gg/GWpVpekp) On 🤗Huggingface: [TeamTonic](https://huggingface.co/TeamTonic) & [MultiTransformer](https://huggingface.co/MultiTransformer) On 🌐Github: [Polytonic](https://github.com/tonic-ai) & contribute to 🌟 [PolyGPT](https://github.com/tonic-ai/polygpt-alpha)"             
@@ -75,7 +73,7 @@ languages = {
 # Global variables to hold component references
 components = {}
 dotenv.load_dotenv()
-seamless_client = Client("facebook/seamless_m4t")
+seamless_client = Client("https://facebook-seamless-m4t-v2-large.hf.space/--replicas/j95rl/")
 HuggingFace_Token = os.getenv("HuggingFace_Token")
 hf_token = os.getenv("HuggingFace_Token")
 base_model_id = os.getenv('BASE_MODEL_ID', 'default_base_model_id')
@@ -170,99 +168,51 @@ def save_image(image_input, output_dir="saved_images"):
         raise ValueError("Invalid image input type")
 
 def process_image(image_file_path):
-    client = Client("https://adept-fuyu-8b-demo.hf.space/--replicas/pqjvl/")
-
-    """
-    Process the image using the Gradio client.
-    """
+    client = Client("https://tonic1-official-qwen-vl-chat.hf.space/--replicas/xhs6q/") # TruEra
     try:
-        # Use the Gradio client to predict
-        result = client.predict(
-            image_file_path,  # File path of the image
-            True,            # Enable detailed captioning
-            fn_index=2       # Function index for the Gradio model
+       result = client.predict(
+            "Describe this image in detail, identify every detail in this image. Describe the image the best you can.",  # TruEra
+            image_file_path,
+            fn_index=0
         )
         return result
     except Exception as e:
         return f"Error occurred during image processing: {e}"
+def process_speech(audio_input, source_language, target_language="English"):
 
-def process_speech(input_language, audio_input):
-    """
-    processing sound using seamless_m4t
-    """
     if audio_input is None:
-        return "no audio or audio did not save yet \nplease try again ! "
-    print(f"audio : {audio_input}")
-    print(f"audio type : {type(audio_input)}")
-    out = seamless_client.predict(
-        "S2TT",
-        "file",
-        None,
-        audio_input,
-        "",
-        input_language,
-        "English",
-        api_name="/run",
-    )
-    out = out[1]  # get the text
-    try:
-        return f"{out}"
-    except Exception as e:
-        return f"{e}"
-
-
-def is_base64(s):
-    try:
-        return base64.b64encode(base64.b64decode(s)) == s.encode()
-    except Exception:
-        return False
-
-def convert_text_to_speech(input_text: str, source_language: str, target_language: str) -> tuple[str, str]:
-    client = Client("https://facebook-seamless-m4t.hf.space/--replicas/8cllp/")
-
+        return "No audio input provided."
     try:
-        # Make a prediction request to the client
-        result = client.predict(
-            "T2ST",
-            "text",  # Since we are doing text-to-speech
-            None,
-            None,
-            input_text,
+        # Predict using the client
+        result = seamless_client.predict(
+            audio_input,  # File path of the audio
             source_language,
             target_language,
-            api_name="/run"
+            api_name="/s2tt"
         )
+        return result
+    except Exception as e:
+        return f"Error in speech processing: {str(e)}"
 
-        # Print or log the raw API response for inspection
-        print("Raw API Response:", result)
-
-        # Initialize variables
-        translated_text = ""
-        audio_file_path = ""
-
-        # Process the result
-        if result:
-            for item in result:
-                if isinstance(item, str):
-                    # Check if the item is a URL pointing to an audio file or a base64 encoded string
-                    if any(ext in item.lower() for ext in ['.mp3', '.wav', '.ogg']) or is_base64(item):
-                        if not audio_file_path:  # Store only the first audio file path or base64 string
-                            audio_file_path = item
-                    else:
-                        # Concatenate the translated text
-                        translated_text += item + " "
+def convert_text_to_speech(input_text, source_language, target_language):
 
-        return audio_file_path, translated_text.strip()
+    try:
+        result = seamless_client.predict(
+            input_text,  
+            source_language, 
+            target_language,  
+            api_name="/t2st"
+        )
+        audio_file_path = result[0] if result else None
+        translated_text = result[1] if result else ""
 
+        return audio_file_path, translated_text
     except Exception as e:
-        print(f"Error in text-to-speech conversion: {str(e)}")
         return None, f"Error in text-to-speech conversion: {str(e)}"
 
 
 def query_vectara(text):
     user_message = text
-
-    # Read authentication parameters from the .env file
     customer_id = os.getenv('CUSTOMER_ID')
     corpus_id = os.getenv('CORPUS_ID')
     api_key = os.getenv('API_KEY')
@@ -371,53 +321,38 @@ def wrap_text(text, width=90):
     wrapped_text = '\n'.join(wrapped_lines)
     return wrapped_text
 
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-1_8B-Chat", trust_remote_code=True) # TruEra
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-1_8B-Chat", device_map="auto", trust_remote_code=True).eval()
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model.to(device)
 
-def multimodal_prompt(user_input, system_prompt="You are an expert medical analyst:"):
-
-    # Combine user input and system prompt
+def multimodal_prompt(user_input, system_prompt="You are an expert medical analyst:"): # TruEra
     formatted_input = f"{user_input}{system_prompt}"
 
     # Encode the input text
-    encodeds = tokenizer(formatted_input, return_tensors="pt", add_special_tokens=False)
-    model_inputs = encodeds.to(device)
+    encoded_input = tokenizer(formatted_input, return_tensors="pt").to(device)
 
-    # Generate a response using the model //MODEL UNDEFINED, using peft_model instead.
-    output = peft_model.generate(
-        **model_inputs,
+    # Generate a response using the model
+    output = model.generate(
+        **encoded_input,
         max_length=512,
         use_cache=True,
         early_stopping=True,
-        bos_token_id=peft_model.config.bos_token_id,
-        eos_token_id=peft_model.config.eos_token_id,
-        pad_token_id=peft_model.config.eos_token_id,
+        pad_token_id=tokenizer.eos_token_id,
         temperature=0.1,
         do_sample=True
     )
 
-    # Decode the response
     response_text = tokenizer.decode(output[0], skip_special_tokens=True)
 
     return response_text
 
-
-# Instantiate the Tokenizer
-tokenizer = AutoTokenizer.from_pretrained("stabilityai/stablelm-3b-4e1t", token=hf_token, trust_remote_code=True, padding_side="left")
-# tokenizer = AutoTokenizer.from_pretrained("Tonic/stablemed", trust_remote_code=True, padding_side="left")
-tokenizer.pad_token = tokenizer.eos_token
-tokenizer.padding_side = 'left'
-
-# Load the PEFT model
-peft_config = PeftConfig.from_pretrained("Tonic/stablemed", token=hf_token)
-peft_model = AutoModelForCausalLM.from_pretrained("stabilityai/stablelm-3b-4e1t", token=hf_token, trust_remote_code=True)
-peft_model = PeftModel.from_pretrained(peft_model, "Tonic/stablemed", token=hf_token)
-
-
 class ChatBot:
     def __init__(self):
         self.history = []
         
     @staticmethod
-    def doctor(user_input, system_prompt="You are an expert medical analyst:"):
+    def doctor(user_input, system_prompt="You are an expert medical analyst:"): # TruEra
         formatted_input = f"{system_prompt}{user_input}"
         user_input_ids = tokenizer.encode(formatted_input, return_tensors="pt")
         response = peft_model.generate(input_ids=user_input_ids, max_length=512, pad_token_id=tokenizer.eos_token_id)
@@ -428,13 +363,11 @@ class ChatBot:
 bot = ChatBot()
 
 
-def process_summary_with_stablemed(summary):
+def process_summary_with_qwen(summary): # TruEra
     system_prompt = "You are a medical instructor . Assess and describe the proper options to your students in minute detail. Propose a course of action for them to base their recommendations on based on your description."
     response_text = bot.doctor(summary, system_prompt)
     return response_text
 
-    
-# Main function to handle the Gradio interface logic
 
 def process_and_query(input_language=None, audio_input=None, image_input=None, text_input=None):
     try:
@@ -492,18 +425,18 @@ def process_and_query(input_language=None, audio_input=None, image_input=None, t
         for source in sources_info:
             markdown_output += f"* {source}\n"
 
-        # Process the summary with Stablemed
-        final_response = process_summary_with_stablemed(summary)
+        # Process the summary with Qwen
+        final_response = process_summary_with_qwen(summary)
 
         # Convert translated text to speech and get both audio file and text
-        target_language = "English"  # Set the target language for the speech
+        target_language = "English"  
         audio_output, translated_text = convert_text_to_speech(final_response, target_language, input_language)
         
         # Evaluate hallucination
         hallucination_label = evaluate_hallucination(final_response, summary)
 
         # Add final response and hallucination label to Markdown output
-        markdown_output += "\n### Processed Summary with StableMed\n"
+        markdown_output += "\n### Processed Summary with Qwen\n"
         markdown_output += final_response + "\n"
         markdown_output += "\n### Hallucination Evaluation\n"
         markdown_output += f"* **Label**: {hallucination_label}\n"
@@ -517,12 +450,10 @@ def process_and_query(input_language=None, audio_input=None, image_input=None, t
 
 
 def clear():
-    # Return default values for each component
     return "English", None, None, "", None
 
 
 def create_interface():
-    # with gr.Blocks(theme='ParityError/Anime') as iface:
     with gr.Blocks(theme='ParityError/Anime') as interface:
         # Display the welcome message
         gr.Markdown(welcome_message)