Update app.py

app.py

@@ -35,14 +35,6 @@ def load_and_classify_csv(file, text_field, event_model):
     not_related = gr.CheckboxGroup(choices=df[df["model_label"]=="none"][text_field].to_list())
     
     return flood_related, fire_related, not_related, model_confidence, len(df[text_field].to_list()), df
-    
-def qa_process(selections):
-    selected_texts = selections
-    
-    analysis_results = [f"Word Count: {len(text.split())}" for text in selected_texts]
-    
-    result_df = pd.DataFrame({"Selected Text": selected_texts, "Analysis": analysis_results})
-    return result_df
 
 def calculate_accuracy(flood_selections, fire_selections, none_selections, num_posts, text_field, data_df):
     posts = data_df[text_field].to_list()
@@ -94,6 +86,42 @@ def add_query(to_add, history):
     if to_add not in history:
         history.append(to_add)
     return gr.CheckboxGroup(choices=history), history
+
+def qa_process(selected_queries, qa_llm_model, aggregator,
+               batch_size, topk, text_field, data_df):
+    
+    emb_model = 'multi-qa-mpnet-base-dot-v1'
+    contexts = []
+
+    queries_df = pd.DataFrame({'id':[j for j in range(len(selected_queries))],'query': selected_queries})
+
+    tweets_df = data_df[[text_field]]
+    tweets_df.reset_index(inplace=True)
+    tweets_df.rename(columns={"index": "order"},inplace=True)
+
+    gr.Info("Loading GENRA pipeline....")
+    genra = GenraPipeline(qa_llm_model, emb_model, aggregator, contexts)
+    gr.Info("Waiting for data...")
+    batches = [tweets_df[i:i+batch_size] for i in range(0,len(tweets_df),batch_size)]
+    
+    genra_answers = []
+    summarize_batch = True
+    for batch_number, tweets in enumerate(batches):
+        gr.Info(f"Populating index for batch {batch_number}")
+        genra.qa_indexer.index_dataframe(tweets)
+        gr.Info(f"Performing retrieval for batch {batch_number}")
+        genra.retrieval(batch_number, queries_df, topk, summarize_batch) 
+    
+    gr.Info("Processed all batches!")
+    # result ------ genra.answers_store
+
+    summary = genra.summarize_history(queries_df)
+    
+    analysis_results = [f"Word Count: {len(text.split())}" for text in selected_queries]
+    
+    result_df = pd.DataFrame({"Selected Text": selected_queries, "Analysis": analysis_results})
+    return result_df, summary
+
             
 with gr.Blocks() as demo:
     event_models = ["jayebaku/distilbert-base-multilingual-cased-crexdata-relevance-classifier"]
@@ -209,7 +237,9 @@ with gr.Blocks() as demo:
 
 
         addqry_button.click(add_query, inputs=[query_inp, queries_state], outputs=[selected_queries, queries_state])
-        qa_button.click(qa_process, inputs=selected_queries, outputs=analysis_output)
-    
+        qa_button.click(qa_process, 
+                        inputs=[selected_queries, qa_llm_model, aggregator, batch_size, topk, text_field, data], 
+                        outputs=[analysis_output, ])
+
 
 demo.launch()