Added selector of p_value

app.py

@@ -133,6 +133,8 @@ def process_submission(*inputs):
         submit_prompt = gr.update(visible=True)
         submission_btn_yes = gr.update(interactive=True, visible=True)
     
+    # TODO: checkbox use_corrected_p_value
+    
     pre_submit_leaderboard_table = gr.update(
         value=leaderboard_server.get_leaderboard(pre_submit=pre_submit, category=leaderboard_server.TASKS_CATEGORY_OVERALL_DETAILS),
         visible=True,
@@ -254,25 +256,29 @@ def fetch_model_detail(submission_id):
         gr.update(value=metadata['link_to_model'], visible=True)
     )
 
-def fetch_model_tournament_results_table(submission_id, category):
+def fetch_model_tournament_results_table(submission_id, category, use_corrected_p_value):
+    kind_of_p_value = leaderboard_server.KINDS_OF_P_VALUE[int(use_corrected_p_value)]
+    
     if submission_id == None or category == None:
         return gr.update(
             visible=False,
         )
     else:
         return gr.update(
-            value=leaderboard_server.get_model_tournament_table(submission_id, category),
+            value=leaderboard_server.get_model_tournament_table(submission_id, category, kind_of_p_value=kind_of_p_value),
             visible=True,
         )
 
-def fetch_model_tournament_results_table_csv(submission_id, category):
+def fetch_model_tournament_results_table_csv(submission_id, category, use_corrected_p_value):
+    kind_of_p_value = leaderboard_server.KINDS_OF_P_VALUE[int(use_corrected_p_value)]
+    
     if submission_id == None or category == None:
         return gr.update(
             visible=False,
         )
     else:
         return gr.update(
-            value=leaderboard_server.get_model_tournament_table_csv(submission_id, category),
+            value=leaderboard_server.get_model_tournament_table_csv(submission_id, category, kind_of_p_value=kind_of_p_value),
             visible=True,
         )
 
@@ -288,7 +294,7 @@ def create_task_abbreviation_legend_table(category):
     
     return task_abbreviation_legend_body
 
-def change_leaderboard_category(category, selected_submission_id):
+def change_leaderboard_category(category, use_corrected_p_value, selected_submission_id):
     if category == leaderboard_server.TASKS_CATEGORY_OVERALL:
         task_abbreviation_legend = gr.update(
             visible=False,
@@ -319,19 +325,21 @@ def change_leaderboard_category(category, selected_submission_id):
             visible=True,
         )
         
-        model_tournament_results_table = fetch_model_tournament_results_table(selected_submission_id, category)
-        model_tournament_results_table_csv = fetch_model_tournament_results_table_csv(selected_submission_id, category)
+        model_tournament_results_table = fetch_model_tournament_results_table(selected_submission_id, category, use_corrected_p_value)
+        model_tournament_results_table_csv = fetch_model_tournament_results_table_csv(selected_submission_id, category, use_corrected_p_value)
+    
+    kind_of_p_value = leaderboard_server.KINDS_OF_P_VALUE[int(use_corrected_p_value)]
     
     leaderboard = gr.update(
-        value=leaderboard_server.get_leaderboard(category=category),
+        value=leaderboard_server.get_leaderboard(category=category, kind_of_p_value=kind_of_p_value),
         visible=True,
     )
     leaderboard_csv = gr.update(
-        value=leaderboard_server.get_leaderboard_csv(category=category),
+        value=leaderboard_server.get_leaderboard_csv(category=category, kind_of_p_value=kind_of_p_value),
         visible=True,
     )
     leaderboard_scatter_plot = gr.update(
-        value=leaderboard_server.get_leaderboard_scatter_plot(category=category),
+        value=leaderboard_server.get_leaderboard_scatter_plot(category=category, kind_of_p_value=kind_of_p_value),
         visible=True,
     )
     
@@ -552,6 +560,9 @@ const intervalId = setInterval(addTitleForEachRowOfLeaderboardTable, 1000);
 </script>
 """
 
+CHECKBOX_USE_CORRECTED_P_VALUE_INFO = "Switch to False Discovery Rate (FDR) guarantees"
+CHECKBOX_USE_CORRECTED_P_VALUE_LABEL = "FDR guarantees"
+
 def gradio_app():
     with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css, head=custom_js) as main:
         check_significance_is_reachable_timer = gr.Timer(
@@ -654,6 +665,13 @@ def gradio_app():
                             interactive=True,
                         )
                     
+                    with gr.Row():
+                        leaderboard_use_corrected_p_value = gr.Checkbox(
+                            info=CHECKBOX_USE_CORRECTED_P_VALUE_INFO,
+                            label=CHECKBOX_USE_CORRECTED_P_VALUE_LABEL,
+                            interactive=True,
+                        )
+                    
                     with gr.Row():
                         leaderboard_table = gr.DataFrame(
                             leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL),
@@ -725,29 +743,35 @@ def gradio_app():
                             visible=False,
                         )
                     
-                    leaderboard_category_of_tasks.change(
-                        fn=change_leaderboard_category,
-                        inputs=[
-                            leaderboard_category_of_tasks,
-                            tournament_results_dropdown,
-                        ],
-                        outputs=[
-                            leaderboard_table,
-                            leaderboard_table_csv,
-                            leaderboard_table_legend,
-                            leaderboard_scatter_plot,
-                            tournament_results_title,
-                            tournament_results_dropdown,
-                            model_tournament_results_table,
-                            model_tournament_results_table_csv,
-                        ],
-                    )
+                    for _leaderboard_form_input in [
+                        leaderboard_category_of_tasks,
+                        leaderboard_use_corrected_p_value,
+                    ]:
+                        _leaderboard_form_input.change(
+                            fn=change_leaderboard_category,
+                            inputs=[
+                                leaderboard_category_of_tasks,
+                                leaderboard_use_corrected_p_value,
+                                tournament_results_dropdown,
+                            ],
+                            outputs=[
+                                leaderboard_table,
+                                leaderboard_table_csv,
+                                leaderboard_table_legend,
+                                leaderboard_scatter_plot,
+                                tournament_results_title,
+                                tournament_results_dropdown,
+                                model_tournament_results_table,
+                                model_tournament_results_table_csv,
+                            ],
+                        )
                     
                     tournament_results_dropdown.change(
                         fn=fetch_model_tournament_results_table,
                         inputs=[
                             tournament_results_dropdown,
                             leaderboard_category_of_tasks,
+                            leaderboard_use_corrected_p_value,
                         ],
                         outputs=model_tournament_results_table,
                     ).then(
@@ -755,6 +779,7 @@ def gradio_app():
                         inputs=[
                             tournament_results_dropdown,
                             leaderboard_category_of_tasks,
+                            leaderboard_use_corrected_p_value,
                         ],
                         outputs=model_tournament_results_table_csv,
                     )
@@ -792,6 +817,14 @@ def gradio_app():
                         interactive=True,
                     )
                 
+                with gr.Row():
+                    model_details_use_corrected_p_value = gr.Checkbox(
+                        info=CHECKBOX_USE_CORRECTED_P_VALUE_INFO,
+                        label=CHECKBOX_USE_CORRECTED_P_VALUE_LABEL,
+                        visible=False,
+                        interactive=True,
+                    )
+                
                 with gr.Row():
                     model_details_model_tournament_results_table = gr.DataFrame(
                         value=None,
@@ -820,11 +853,16 @@ def gradio_app():
                     fn=lambda submission_id: gr.update(visible=True) if submission_id else gr.update(visible=False),
                     inputs=model_details_model_dropdown,
                     outputs=model_details_category_of_tasks
+                ).then(
+                    fn=lambda submission_id: gr.update(visible=True) if submission_id else gr.update(visible=False),
+                    inputs=model_details_model_dropdown,
+                    outputs=model_details_use_corrected_p_value
                 ).then(
                     fn=fetch_model_tournament_results_table,
                     inputs=[
                         model_details_model_dropdown,
                         model_details_category_of_tasks,
+                        model_details_use_corrected_p_value,
                     ],
                     outputs=model_details_model_tournament_results_table
                 ).then(
@@ -832,25 +870,32 @@ def gradio_app():
                     inputs=[
                         model_details_model_dropdown,
                         model_details_category_of_tasks,
+                        model_details_use_corrected_p_value,
                     ],
                     outputs=model_details_model_tournament_results_table_csv
                 )
                 
-                model_details_category_of_tasks.change(
-                    fn=fetch_model_tournament_results_table,
-                    inputs=[
-                        model_details_model_dropdown,
-                        model_details_category_of_tasks,
-                    ],
-                    outputs=model_details_model_tournament_results_table,
-                ).then(
-                    fn=fetch_model_tournament_results_table_csv,
-                    inputs=[
-                        model_details_model_dropdown,
-                        model_details_category_of_tasks,
-                    ],
-                    outputs=model_details_model_tournament_results_table_csv,
-                )
+                for _model_details_form_input in [
+                    model_details_category_of_tasks,
+                    model_details_use_corrected_p_value,
+                ]:
+                    _model_details_form_input.change(
+                        fn=fetch_model_tournament_results_table,
+                        inputs=[
+                            model_details_model_dropdown,
+                            model_details_category_of_tasks,
+                            model_details_use_corrected_p_value,
+                        ],
+                        outputs=model_details_model_tournament_results_table,
+                    ).then(
+                        fn=fetch_model_tournament_results_table_csv,
+                        inputs=[
+                            model_details_model_dropdown,
+                            model_details_category_of_tasks,
+                            model_details_use_corrected_p_value,
+                        ],
+                        outputs=model_details_model_tournament_results_table_csv,
+                    )
             
             with gr.TabItem('Submission'):
                 with gr.Column():

server.py

@@ -277,6 +277,8 @@ class LeaderboardServer:
         self.CATEGORY_TO_TASK_ABBREVIATION_TO_DETAILS = self._prepare_category_to_task_abbr_to_details()
         self.MAX_LENGTH_OF_MODEL_TITLE = 28
         self.DIR_DATAFRAMES_CSV = "./dataframes_csv"
+        self.DEFAULT_KIND_OF_P_VALUE = "p_value"
+        self.KINDS_OF_P_VALUE = ["p_value", "corrected_p_value"]
         
         self.var_lock = ReadWriteLock()
         self.submission_ids = set()
@@ -326,45 +328,45 @@ class LeaderboardServer:
         
         categories = [self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS] + sorted(self.TASKS_CATEGORIES)
         
-        leaderboard_dataframes = {
-            category: self._get_leaderboard(category=category) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
+        leaderboard_dataframes = {kind_of_p_value: {
+            category: self._get_leaderboard(category=category, kind_of_p_value=kind_of_p_value) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
             for category in categories
-        }
+        } for kind_of_p_value in self.KINDS_OF_P_VALUE}
         
         with self.var_lock.ro:
             submission_ids = self.submission_ids
         
-        tournament_dataframes = {
+        tournament_dataframes = {kind_of_p_value: {
             submission_id: {
-                category: self._get_model_tournament_table(submission_id, category) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
+                category: self._get_model_tournament_table(submission_id, category, kind_of_p_value=kind_of_p_value) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity'])
                 for category in categories
             }
             for submission_id in submission_ids
-        }
+        } for kind_of_p_value in self.KINDS_OF_P_VALUE}
         
         with self.var_lock.rw:
             self.leaderboard_dataframes = leaderboard_dataframes
             self.tournament_dataframes = tournament_dataframes
         
-        leaderboard_dataframes_csv = {
+        leaderboard_dataframes_csv = {kind_of_p_value: {
             category: self._dataframe_to_csv(
-                self._get_leaderboard(category=category, to_csv=True) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity']),
-                f"Leaderboard - {category}.csv"
+                self._get_leaderboard(category=category, to_csv=True, kind_of_p_value=kind_of_p_value) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity']),
+                f"Leaderboard - {category}{' - FDR guarantees' if kind_of_p_value != self.DEFAULT_KIND_OF_P_VALUE else ''}.csv"
             )
             for category in categories
-        }
+        } for kind_of_p_value in self.KINDS_OF_P_VALUE}
         
         with self.var_lock.ro:
-            tournament_dataframes_csv = {
+            tournament_dataframes_csv = {kind_of_p_value: {
                 submission_id: {
                     category: self._dataframe_to_csv(
-                        self._get_model_tournament_table(submission_id, category, to_csv=True) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity']),
-                        f"Tournament table - {self.submission_id_to_data[submission_id]['submission_metadata']['model_name'][:self.MAX_LENGTH_OF_MODEL_TITLE].replace('/', '_')} - {category}.csv",
+                        self._get_model_tournament_table(submission_id, category, to_csv=True, kind_of_p_value=kind_of_p_value) if not self.tournament_results_corrupted else pd.DataFrame(columns=['Corrupted, please check integrity']),
+                        f"Tournament table - {self.submission_id_to_data[submission_id]['submission_metadata']['model_name'][:self.MAX_LENGTH_OF_MODEL_TITLE].replace('/', '_')} - {category}{' - FDR guarantees' if kind_of_p_value != self.DEFAULT_KIND_OF_P_VALUE else ''}.csv",
                     )
                     for category in categories
                 }
                 for submission_id in submission_ids
-            }
+            } for kind_of_p_value in self.KINDS_OF_P_VALUE}
         
         with self.var_lock.rw:
             self.leaderboard_dataframes_csv = leaderboard_dataframes_csv
@@ -554,30 +556,36 @@ class LeaderboardServer:
               df_css.loc[i, c] = ''
         return df_css 
 
-    def get_model_tournament_table_csv(self, submission_id, category, pre_submit=None):
+    def get_model_tournament_table_csv(self, submission_id, category, pre_submit=None, kind_of_p_value=None):
+        kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
+        
         if pre_submit == None:
             with self.var_lock.ro:
-                return self.tournament_dataframes_csv[submission_id][category]
+                return self.tournament_dataframes_csv[kind_of_p_value][submission_id][category]
         else:
             return self._dataframe_to_csv(
-                self._get_model_tournament_table(submission_id, category, pre_submit=pre_submit, to_csv=True),
+                self._get_model_tournament_table(submission_id, category, pre_submit=pre_submit, to_csv=True, kind_of_p_value=kind_of_p_value),
                 f"Tournament table - pre-submit - {category}.csv",
             )
 
-    def get_model_tournament_table(self, submission_id, category, pre_submit=None):
+    def get_model_tournament_table(self, submission_id, category, pre_submit=None, kind_of_p_value=None):
+        kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
+        
         if pre_submit == None:
             with self.var_lock.ro:
-                return copy.copy(self.tournament_dataframes[submission_id][category])
+                return copy.copy(self.tournament_dataframes[kind_of_p_value][submission_id][category])
         else:
-            return self._get_model_tournament_table(submission_id, category, pre_submit=pre_submit)
+            return self._get_model_tournament_table(submission_id, category, pre_submit=pre_submit, kind_of_p_value=kind_of_p_value)
 
-    def _get_model_tournament_table(self, submission_id, category, pre_submit=None, to_csv=False):
+    def _get_model_tournament_table(self, submission_id, category, pre_submit=None, to_csv=False, kind_of_p_value=None):
+        kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
+        
         model_tournament_table = []
         
         with self.var_lock.ro:
             tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
             
-            for competitor_id in tournament_results[submission_id].keys() - {submission_id}: # without self
+            for competitor_id in tournament_results[submission_id].keys() - {submission_id}:  # without self
                 if competitor_id not in self.submission_id_to_data:
                     if pre_submit and competitor_id == pre_submit.submission_id:
                         data = pre_submit.data
@@ -590,13 +598,14 @@ class LeaderboardServer:
                 for task in self.TASKS_METADATA:
                     task_category = self.TASKS_METADATA[task]["category"]
                     if category in (task_category, self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
+                        match_task_result_details = dict.fromkeys(["significant", "corrected_p_value", "p_value"])  # order has impact to sorting DataFrame
+                        match_task_result_details.update(copy.deepcopy(tournament_results[submission_id][competitor_id][task]))
+                        match_task_result_significant = self._is_task_pval_significant(match_task_result_details[kind_of_p_value])
+                        
                         if to_csv:
-                            match_results[task] = tournament_results[submission_id][competitor_id][task]["significant"]
+                            match_results[task] = match_task_result_significant
                         else:
-                            match_task_result_details = dict.fromkeys(["significant", "corrected_p_value", "p_value"])  # order has impact to sorting DataFrame
-                            match_task_result_details.update(copy.deepcopy(tournament_results[submission_id][competitor_id][task]))
-                            match_task_result_details["significant"] = str(match_task_result_details["significant"]).lower()  # originaly bool
-                            match_task_result_significant = match_task_result_details["significant"]
+                            match_task_result_details["significant"] = str(match_task_result_significant).lower()  # originaly bool
                             match_task_result_details = "\n".join(f"{k}: {v}" for k, v in match_task_result_details.items())
                             match_results[task] = f'<abbr title={xmlQuoteAttr(match_task_result_details)}>{match_task_result_significant}</abbr>'
                 
@@ -654,7 +663,10 @@ class LeaderboardServer:
         
         return True
 
-    def _correct_significance_in_tournament_results(self, tournament_results, alpha=0.05):
+    def _is_task_pval_significant(self, task_pval, alpha=0.05):
+        return bool(task_pval < alpha)
+
+    def _correct_significance_in_tournament_results(self, tournament_results):
         tournament_results = copy.deepcopy(tournament_results)
         
         if not self._is_correct_significance_in_tournament_results(tournament_results):
@@ -665,7 +677,7 @@ class LeaderboardServer:
                     corrected_model_task_pvals = correct_pvals_for_fdr(model_task_pvals)
                     for competitor_id, task_pval in zip(competitors, corrected_model_task_pvals):
                         tournament_results[submission_id][competitor_id][task]["corrected_p_value"] = task_pval
-                        tournament_results[submission_id][competitor_id][task]["significant"] = bool(task_pval < alpha)
+                        tournament_results[submission_id][competitor_id][task]["significant"] = self._is_task_pval_significant(task_pval)
         
         return tournament_results
 
@@ -680,17 +692,19 @@ class LeaderboardServer:
         dataframe.to_csv(filepath, index=False)
         return filepath
 
-    def get_leaderboard_scatter_plot(self, pre_submit=None, category=None):
+    def get_leaderboard_scatter_plot(self, pre_submit=None, category=None, kind_of_p_value=None):
         import numpy as np
         from analyze_winscore import get_ldb_records, create_scatter_plot_with_curve_with_variances_named
         
+        kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
+        
         #m = self.TASKS_METADATA
         #tournament = self.tournament_results
         name_map = self.submission_id_to_model_title
         
         category = category if category else self.TASKS_CATEGORY_OVERALL
         
-        csv_file_path = self.leaderboard_dataframes_csv[self.TASKS_CATEGORY_OVERALL]
+        csv_file_path = self.leaderboard_dataframes_csv[kind_of_p_value][self.TASKS_CATEGORY_OVERALL]
         ldb_records = get_ldb_records(name_map, csv_file_path)
         categories = self.TASKS_CATEGORIES
         model_names = list(ldb_records.keys())
@@ -725,29 +739,32 @@ class LeaderboardServer:
         
         return fig
 
-    def get_leaderboard_csv(self, pre_submit=None, category=None):
+    def get_leaderboard_csv(self, pre_submit=None, category=None, kind_of_p_value=None):
         if pre_submit == None:
             category = category if category else self.TASKS_CATEGORY_OVERALL
+            kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
             with self.var_lock.ro:
-                return self.leaderboard_dataframes_csv[category]
+                return self.leaderboard_dataframes_csv[kind_of_p_value][category]
         else:
             return self._dataframe_to_csv(
-                self._get_leaderboard(pre_submit=pre_submit, category=category, to_csv=True),
+                self._get_leaderboard(pre_submit=pre_submit, category=category, to_csv=True, kind_of_p_value=kind_of_p_value),
                 f"Leaderboard - pre-submit - {category}.csv",
             )
 
-    def get_leaderboard(self, pre_submit=None, category=None):
+    def get_leaderboard(self, pre_submit=None, category=None, kind_of_p_value=None):
         if pre_submit == None:
             category = category if category else self.TASKS_CATEGORY_OVERALL
+            kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
             with self.var_lock.ro:
-                return copy.copy(self.leaderboard_dataframes[category])
+                return copy.copy(self.leaderboard_dataframes[kind_of_p_value][category])
         else:
-            return self._get_leaderboard(pre_submit=pre_submit, category=category)
+            return self._get_leaderboard(pre_submit=pre_submit, category=category, kind_of_p_value=kind_of_p_value)
 
-    def _get_leaderboard(self, pre_submit=None, category=None, to_csv=False):
+    def _get_leaderboard(self, pre_submit=None, category=None, to_csv=False, kind_of_p_value=None):
         with self.var_lock.ro:
             tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
             category = category if category else self.TASKS_CATEGORY_OVERALL
+            kind_of_p_value = kind_of_p_value if kind_of_p_value else self.DEFAULT_KIND_OF_P_VALUE
 
             if len(tournament_results) == 0:
                 return pd.DataFrame(columns=['No submissions yet'])
@@ -779,8 +796,13 @@ class LeaderboardServer:
                             num_of_wins = 0
                             for competitor_id in tournament_results[submission_id].keys() - {submission_id}: # without self
                                 num_of_competitors += 1
-                                if tournament_results[submission_id][competitor_id][task]["significant"]:
+                                
+                                match_task_result_details = tournament_results[submission_id][competitor_id][task]
+                                match_task_result_significant = self._is_task_pval_significant(match_task_result_details[kind_of_p_value])
+                                
+                                if match_task_result_significant:
                                     num_of_wins += 1
+                            
                             task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
                             win_score.setdefault(task_category, []).append(task_score)
                             
@@ -1061,7 +1083,7 @@ class LeaderboardServer:
                         print(f"Locked `submit_lock` for {submission_id = }")
                         print(info_msg)
                         
-                    self.update_leaderboard()
+                    self.update_leaderboard()  # TODO: Přidat komentář proč to tady je. Nemělo by to být pouze při `do_submit == True`?
                     
                     if HF_FAKE_TOURNAMENT:
                         tournament_results = self.fake_tournament(submission_id, file)