implment changes for new design and count summary of test info

app.py

@@ -24,17 +24,19 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     # shutil.copyfile(label.name, saved_test_label)
     # shutil.copyfile(info.name, saved_train_info)
     parent_location="ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/"
-    if(model_name=="High Graduated Schools"):
+    test_info_location=parent_location+"fullTest/test_info.txt"
+    test_location=parent_location+"fullTest/test.txt"
+    if(model_name=="ASTRA-FT-HGR"):
         finetune_task="highGRschool10"
-        test_info_location=parent_location+"highGRschool10/test_info.txt"
-        test_location=parent_location+"highGRschool10/test.txt"
-    elif(model_name== "Low Graduated Schools" ):
+        # test_info_location=parent_location+"fullTest/test_info.txt"
+        # test_location=parent_location+"fullTest/test.txt"
+    elif(model_name== "ASTRA-FT-LGR" ):
         finetune_task="lowGRschoolAll"
-        test_info_location=parent_location+"lowGRschoolAll/test_info.txt"
-        test_location=parent_location+"lowGRschoolAll/test.txt"
-    elif(model_name=="Full Set"):
-        test_info_location=parent_location+"highGRschool10/test_info.txt"
-        test_location=parent_location+"highGRschool10/test.txt"
+        # test_info_location=parent_location+"lowGRschoolAll/test_info.txt"
+        # test_location=parent_location+"lowGRschoolAll/test.txt"
+    elif(model_name=="ASTRA-FT-FULL"):
+        # test_info_location=parent_location+"fullTest/test_info.txt"
+        # test_location=parent_location+"fullTest/test.txt"
         finetune_task="highGRschool10"
     else:
         finetune_task=None
@@ -83,7 +85,74 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     #     model_name="highGRschool10"
     # else:
     #     model_name="highGRschool10"
-    progress(0.2, desc="Executing models")
+    # Function to analyze each row
+    def analyze_row(row):
+        # Split the row into fields
+        fields = row.split("\t")
+
+        # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer
+        optional_task_1_subtasks = ["DenominatorFactor", "NumeratorFactor", "EquationAnswer"]
+        optional_task_2_subtasks = [
+            "FirstRow2:1", "FirstRow2:2", "FirstRow1:1", "FirstRow1:2", 
+            "SecondRow", "ThirdRow"
+        ]
+        final_answer_tasks = ["FinalAnswer"]
+
+        # Helper function to evaluate task attempts
+        def evaluate_tasks(fields, tasks):
+            task_status = {}
+            for task in tasks:
+                relevant_attempts = [f for f in fields if task in f]
+                # print(relevant_attempts)
+                if any("OK" in attempt for attempt in relevant_attempts):
+                    task_status[task] = "Attempted (Successful)"
+                elif any("ERROR" in attempt for attempt in relevant_attempts):
+                    task_status[task] = "Attempted (Error)"
+                elif any("JIT" in attempt for attempt in relevant_attempts):
+                    task_status[task] = "Attempted (JIT)"
+                else:
+                    task_status[task] = "Unattempted"
+            return task_status
+
+        # Evaluate tasks for each category
+        optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)
+        optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)
+        final_answer_status = evaluate_tasks(fields, final_answer_tasks)
+
+        # Combine results
+        result = {
+            "OptionalTask_1": optional_task_1_status,
+            "OptionalTask_2": optional_task_2_status,
+            "FinalAnswer": final_answer_status,
+        }
+        return result
+    # Read data from test_info.txt
+    with open(test_info_location, "r") as file:
+        data = file.readlines()
+    results = [analyze_row(row.strip()) for row in data if row.strip()]
+
+    status_counts = {}
+
+
+    for result in results:
+        for task_category, task_statuses in result.items():
+            for task, status in task_statuses.items():
+                if task not in status_counts:
+                    status_counts[task] = {"Attempted (Successful)": 0, "Attempted (Error)": 0, 
+                                        "Attempted (JIT)": 0, "Unattempted": 0}
+                status_counts[task][status] += 1
+
+    # Create a string output for results
+    output_summary = "Task Analysis Summary:\n"
+    output_summary += "-----------------------\n"
+
+    for task, statuses in status_counts.items():
+        output_summary += f"Task: {task}\n"
+        for status, count in statuses.items():
+            output_summary += f"  {status}: {count}\n"
+
+
+    progress(0.2, desc="analysis done!! Executing models")
     subprocess.run([
         "python", "new_test_saved_finetuned_model.py",
         "-workspace_name", "ratio_proportion_change3_2223/sch_largest_100-coded",
@@ -113,7 +182,7 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     fig, ax = plt.subplots()
     ax.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')
     ax.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
-    ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'ROC Curve: {model_name}')
+    ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'(Receiver Operating Curve) ROC')
     ax.legend(loc="lower right")
     ax.grid()
 
@@ -127,23 +196,20 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     # Prepare text output with HTML formatting
     text_output = f"""
     Model: {model_name}\n
-    Result Summary:\n
     -----------------\n
-    Precision: {result['precisions']:.2f}\n
-    Recall: {result['recalls']:.2f}\n
+
     Time Taken: {result['time_taken_from_start']:.2f} seconds\n
     Total Schools in test: {len(unique_schools):.4f}\n
     Total Schools taken: {len(random_schools):.4f}\n
-    High grad schools: {len(high_sample):.4f}\n
-    Low grad schools: {len(low_sample):.4f}\n
+    Total number of instances having Schools with HGR : {len(high_sample):.4f}\n
+    Total number of instances having Schools with LGR: {len(low_sample):.4f}\n
     -----------------\n
-    Note: The ROC Curve is also displayed for the evaluation.
     """
-    return text_output,plot_path
+    return text_output,plot_path,output_summary
 
 # List of models for the dropdown menu
 
-models = ["High Graduated Schools", "Low Graduated Schools", "Full Set"]
+models = ["ASTRA-FT-HGR", "ASTRA-FT-LGR", "ASTRA-FT-FULL"]
 
 # Create the Gradio interface
 with gr.Blocks(css="""
@@ -328,7 +394,7 @@ tbody.svelte-18wv37q>tr.svelte-18wv37q:nth-child(odd) {
     color: white;
 """) as demo:
     gr.Markdown("<h1 id='title'>ASTRA</h1>", elem_id="title")
-    gr.Markdown("<p class='description'>Upload a .txt file and select a model from the dropdown menu.</p>")
+    
     
     with gr.Row():
         # file_input = gr.File(label="Upload a test file", file_types=['.txt'], elem_classes="file-box")
@@ -336,18 +402,19 @@ tbody.svelte-18wv37q>tr.svelte-18wv37q:nth-child(odd) {
 
         # info_input = gr.File(label="Upload test info", file_types=['.txt'], elem_classes="file-box")
     
-        model_dropdown = gr.Dropdown(choices=models, label="Select Finetune Task", elem_classes="dropdown-menu")
+        model_dropdown = gr.Dropdown(choices=models, label="Select Fine-tuned Model", elem_classes="dropdown-menu")
 
     
     increment_slider = gr.Slider(minimum=1, maximum=100, step=1, label="Schools Percentage", value=1)
-    
+    gr.Markdown("<p class='description'>Dashboard</p>")
     with gr.Row():
-        output_text = gr.Textbox(label="Output Text")
-        output_image = gr.Image(label="Output Plot")
+        output_text = gr.Textbox(label="")
+        output_image = gr.Image(label="ROC")
+        output_summary = gr.Textbox(label="Summary")
 
     btn = gr.Button("Submit")
  
-    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,output_image])
+    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,output_image,output_summary])
 
 
 # Launch the app

result.txt

@@ -1,7 +1,7 @@
-avg_loss: 0.5569005310535431
-total_acc: 74.30213464696223
-precisions: 0.7660032941165892
-recalls: 0.7430213464696224
-f1_scores: 0.7359098644855878
-time_taken_from_start: 41.834863901138306
-auc_score: 0.7675472675472674
+avg_loss: 0.5820454359054565
+total_acc: 69.02834008097166
+precisions: 0.7233000757179396
+recalls: 0.6902834008097166
+f1_scores: 0.680564448931978
+time_taken_from_start: 37.70641040802002
+auc_score: 0.7527458335895701

roc_data.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c022a6b5eaa8a1a3c8cb6f10578afc01f92a1f9800ec4ebe1ab78b22b3ddd988
-size 10685
+oid sha256:57ef1d1fe518f78a1087a3605e61462c3489cfbd0e2b7cc7bc9cf4b6360b60f7
+size 9917