really fucking annoyed

app.py

@@ -8,11 +8,8 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, AutoProcessor
 import tensorflow as tf
 import tensorflow_hub as hub
 import io
-import os 
-import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
-import tempfile 
-import shutil
+import tempfile # Add this import
 import logging
 
 # Configure logging
@@ -76,43 +73,64 @@ def save_dataframe_to_csv(df):
     # Return the file path (no need to reopen the file with "rb" mode)
     return temp_file_path
 
-def process_images_and_statements(image_file):
-    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-        shutil.copyfileobj(image_file, temp_file)
-
-    image = Image.open(temp_file.name)
-    image = np.array(image)
+# Main function to perform image captioning and image-text matching
+def process_images_and_statements(image):
     logging.info('Starting process_images_and_statements')
 
-    # Generate the image caption
-    generated_caption = caption_image(image)
+    # Generate image caption for the uploaded image using git-large-r-textcaps
+    caption = generate_caption(git_processor_large_textcaps, git_model_large_textcaps, image)
+
+    # Define weights for combining textual similarity score and image-statement ITM score (adjust as needed)
+    weight_textual_similarity = 0.5
+    weight_statement = 0.5
+
+    # Initialize an empty list to store the results
+    results_list = []
+
+    # Loop through each predefined statement
+    for statement in statements:
+        # Compute textual similarity between caption and statement
+        textual_similarity_score = (compute_textual_similarity(caption, statement) * 100) # Multiply by 100
+
+        # Compute ITM score for the image-statement pair
+        itm_score_statement = (compute_itm_score(image, statement) * 100) # Multiply by 100
 
-    # Match the statements
-    matched_statements = match_statements(image, statements)
+        # Combine the two scores using a weighted average
+        final_score = ((weight_textual_similarity * textual_similarity_score) +
+                       (weight_statement * itm_score_statement))
 
-    os.unlink(temp_file.name)  # Remove the temporary file
+        # Append the result to the results_list
+        results_list.append({
+            'Statement': statement,
+            'Generated Caption': caption, # Include the generated caption
+            'Textual Similarity Score': f"{textual_similarity_score:.2f}%", # Format as percentage with two decimal places
+            'ITM Score': f"{itm_score_statement:.2f}%", # Format as percentage with two decimal places
+            'Final Combined Score': f"{final_score:.2f}%" # Format as percentage with two decimal places
+        })
 
-    return generated_caption, matched_statements
+    # Convert the results_list to a DataFrame using pandas.concat
+    results_df = pd.concat([pd.DataFrame([result]) for result in results_list], ignore_index=True)
 
-# Define Gradio interface
-image_input = gr.inputs.Image(type="numpy", label="Upload Image")
-outputs = [
-    gr.outputs.Image(type="pil", label="Annotated Image"),
-    gr.outputs.Textbox(label="Matched Statements"),
-]
+    logging.info('Finished process_images_and_statements')
+
+    # Save results_df to a CSV file
+    csv_results = save_dataframe_to_csv(results_df)
+
+    # Return both the DataFrame and the CSV data for the Gradio interface
+    return results_df, csv_results # <--- Return results_df and csv_results
+
+# Gradio interface
+image_input = gr.inputs.Image()
+output_df = gr.outputs.Dataframe(type="pandas", label="Results")
+output_csv = gr.outputs.File(label="Download CSV")
 
 iface = gr.Interface(
     fn=process_images_and_statements,
     inputs=image_input,
-    outputs=outputs,
-    title="Image Captioning and Matching",
-    description="Upload an image to generate a caption for the image and match the statements.",
-    theme='sudeepshouche/minimalist'
+    outputs=[output_df, output_csv], # Include both the DataFrame and CSV file outputs
+    title="Image Captioning and Image-Text Matching",
+    theme='sudeepshouche/minimalist',
+    css=".output { flex-direction: column; } .output .outputs { width: 100%; }" # Custom CSS
 )
 
-# Launch Gradio app
-iface.launch(debug=True)
-
-
-# Launch Gradio app
-iface.launch(debug=True)
+iface.launch()