[FIX] Read evals

app.py

@@ -262,118 +262,6 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model Types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
-                    filter_domain_specific = gr.CheckboxGroup(
-                        label="Domain Specificity",
-                        choices=["🏥  Clinical models", "Generic models"],
-                        value=["🏥  Clinical models", "Generic models"],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
-
-            leaderboard_table = gr.components.Dataframe(
-                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=datasets_original_df[DATASET_COLS],
-                headers=DATASET_COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-                        
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    search_bar,
-                    filter_columns_type,
-                    filter_domain_specific,
-                    filter_columns_size
-                    # filter_columns_architecture
-                ],
-                leaderboard_table,
-            )
-            for selector in [
-                shown_columns,
-                filter_columns_type,
-                filter_domain_specific,
-                # filter_columns_architecture,
-                filter_columns_size,
-                # deleted_models_visibility,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        search_bar,
-                        filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size
-                        # filter_columns_architecture,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-
         with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
             with gr.Row():
                 with gr.Column():
@@ -1065,6 +953,119 @@ with demo:
                                 leaderboard_table,
                                 queue=True,
                             )
+        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain Specificity",
+                        choices=["🏥  Clinical models", "Generic models"],
+                        value=["🏥  Clinical models", "Generic models"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[DATASET_COLS],
+                headers=DATASET_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                # filter_columns_architecture,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                        # filter_columns_architecture,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=5):
             gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
             gr.HTML(FIVE_PILLAR_DIAGRAM)

src/envs.py

@@ -8,7 +8,7 @@ TOKEN = os.environ.get("TOKEN")  # A read/write token for your org
 
 OWNER = "m42-health"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
-PRIVATE_REPO = True
+PRIVATE_REPO = False
 
 
 if PRIVATE_REPO:

src/leaderboard/read_evals.py

@@ -54,7 +54,8 @@ class EvalResult:
             except:
                 breakpoint()
             
- 
+        # if "deepseek-ai/DeepSeek-R1-Distill-Llama-70B" in json_filepath:
+        #     breakpoint()
         config = data.get("config")
 
         # Precision
@@ -113,7 +114,8 @@ class EvalResult:
             if open_ended_results["ELO_intervals"] is not None and open_ended_results["Score_intervals"] is not None:
                 open_ended_results["ELO_intervals"] = "+" + str(open_ended_results["ELO_intervals"][1]) + "/-" + str(abs(open_ended_results["ELO_intervals"][0]))
                 open_ended_results["Score_intervals"] = "+" + str(open_ended_results["Score_intervals"][1]) + "/-" + str(abs(open_ended_results["Score_intervals"][0]))
-        # breakpoint()
+        # if "deepseek-ai/DeepSeek-R1-Distill-Llama-70B" in json_filepath:
+        #     breakpoint()
         # changes to be made here
         med_safety_results = {}
         if "med-safety" in data["results"]:
@@ -178,12 +180,12 @@ class EvalResult:
                     continue
                 mean_acc = np.mean(accs)  # * 100.0
                 closed_ended_arabic_results[task.benchmark] = mean_acc
-        if open_ended_results == {} or med_safety_results == {} or medical_summarization_results == {} or aci_results == {} or soap_results == {}:
-            open_ended_results = {}
-            med_safety_results = {}
-            medical_summarization_results = {}
-            aci_results = {}
-            soap_results = {}
+        # if open_ended_results == {} or med_safety_results == {} or medical_summarization_results == {} or aci_results == {} or soap_results == {}:
+        #     open_ended_results = {}
+        #     med_safety_results = {}
+        #     medical_summarization_results = {}
+        #     aci_results = {}
+        #     soap_results = {}
         # types_results = {}
         # for clinical_type in ClinicalTypes:
         #     clinical_type = clinical_type.value
@@ -195,7 +197,8 @@ class EvalResult:
 
         #     mean_acc = np.mean(accs)  # * 100.0
         #     types_results[clinical_type.benchmark] = mean_acc
-
+        # if "deepseek-ai/DeepSeek-R1-Distill-Llama-70B" in json_filepath:
+        #     breakpoint()
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -337,6 +340,14 @@ def get_request_file_for_model(requests_path, model_name, precision):
                 request_file = tmp_request_file
     return request_file
 
+def update_results(result1, result2):
+    # breakpoint()
+    for key in dir(result1):
+        if key.endswith("_results"):
+            if getattr(result1, key) == {}:
+                setattr(result1, key, getattr(result2, key))
+    # breakpoint()
+    return result1
 
 def get_raw_eval_results(results_path: str, requests_path: str, evaluation_metric: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
@@ -355,7 +366,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, evaluation_metri
 
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
-
+    # breakpoint()
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
@@ -364,11 +375,12 @@ def get_raw_eval_results(results_path: str, requests_path: str, evaluation_metri
 
         # Store results of same eval together
         eval_name = eval_result.eval_name
-        # if eval_name in eval_results.keys():
-        #     eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        # else:
-        eval_results[eval_name] = eval_result
-
+        if eval_name in eval_results.keys():
+            eval_results[eval_name] = update_results(eval_results[eval_name], eval_result)
+            # eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+    # breakpoint()
     results = []
     # clinical_type_results = []
     for v in eval_results.values():