Merge branch 'results_dataset_integrity_check' into 'more_match_details'

README.md

@@ -1,5 +1,5 @@
 ---
-title: LLM Benchmark
+title: 🇨🇿 BenCzechMark [Beta Preview]
 emoji: 📊
 colorFrom: gray
 colorTo: blue
@@ -9,4 +9,4 @@ app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -255,60 +255,19 @@ tr.row_odd {
   background-color: var(--table-odd-background-fill);
 }
 
-.leaderboard-table th:first-child, .leaderboard-table td:first-child {
-  position: sticky;
-  left: 0;
-  z-index: 1;
-  background-color: inherit;
-}
-
 .leaderboard-table td:first-child p, .leaderboard-table-model-details td:first-child p {
   margin: 0px;
 }
 
-.leaderboard-table th:nth-child(2), .leaderboard-table td:nth-child(2) {
-  position: sticky;
-  left: var(--cell-width-0);
-  z-index: 1;
-  background-color: inherit;
-}
-
-.leaderboard-table th:nth-child(3), .leaderboard-table td:nth-child(3) {
-  position: sticky;
-  left: calc(var(--cell-width-0) + var(--cell-width-1));
-  z-index: 1;
-  background-color: inherit;
-}
-
-.leaderboard-table th:nth-child(4), .leaderboard-table td:nth-child(4) {
-  position: sticky;
-  left: calc(var(--cell-width-0) + var(--cell-width-1) + var(--cell-width-2));
-  z-index: 1;
-  background-color: inherit;
-}
-
 .leaderboard-table th:nth-child(5), .leaderboard-table td:nth-child(5) {
-  position: sticky;
-  left: calc(var(--cell-width-0) + var(--cell-width-1) + var(--cell-width-2) + var(--cell-width-3));
-  z-index: 1;
-  background-color: inherit;
+  border-right-width: 2px;
+  border-right-color: var(--border-color-primary);
 }
 
 .leaderboard-table td:nth-child(5) p {
   font-weight: bolder;
 }
 
-.leaderboard-table th:nth-child(5)::after, .leaderboard-table td:nth-child(5)::after {
-  box-shadow: inset 5px 0px 4px -4px var(--border-color-primary);
-  position: absolute;
-  top: 0;
-  right: 0;
-  bottom: 0;
-  content: "";
-  width: 30px;
-  transform: translateX(100%);
-}
-
 """
 
 with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:

content.py

@@ -2,7 +2,7 @@
 This file contains the text content for the leaderboard client.
 """
 HEADER_MARKDOWN = """
-# ![Czech Republic Flag](https://raw.githubusercontent.com/yammadev/flag-icons/bd4bcf4f4829002cd10416029e05ba89a7554af4/png/CZ.png) BenCzechMark [Beta Preview]
+# 🇨🇿 BenCzechMark [Beta Preview]
 
 Welcome to the leaderboard! Here you can submit your model and compare it with the existing models.
 """

server.py

@@ -161,11 +161,14 @@ class LeaderboardServer:
         self.submission_ids = set()
         self.submission_id_to_file = {}  # Map submission ids to file paths
         self.submission_id_to_model_title = {}
+        self.submission_id_to_data = {}  # Only data (results and metadata) using by leaderboard
         self.fetch_existing_models()
         self.tournament_results = self.load_tournament_results()
         
         self.pre_submit_lock = pre_submit_lock
         self.pre_submit = None
+        
+        self.results_dataset_integrity_check()  # Check integrity of the results dataset after (re)start Hugging Face Space
 
     def update_leaderboard(self):
         self.local_leaderboard = snapshot_download(
@@ -215,6 +218,64 @@ class LeaderboardServer:
                 self.submission_ids.add(submission_id)
                 self.submission_id_to_file[submission_id] = submission_file
                 self.submission_id_to_model_title[submission_id] = metadata["team_name"] + "/" + metadata["model_name"]
+                self.submission_id_to_data[submission_id] = {"results": data["results"], "metadata": metadata}
+
+    def results_dataset_integrity_check(self):
+        """
+        Zkontroluje, že:
+        - všechny modely byly v duelu se všemi
+        -- pokud ne, znemožní potvrzení nových submitů a udělá zbývající zápasy
+        -- kontroluje soubory v adresáři "/data" a soubor "tournament.json"
+        - v souboru "tournament.json" není `submission_id`, které by nemělo soubor v adresáři "/data"
+        """
+        
+        while True:
+            with self.pre_submit_lock:
+                if self.pre_submit == None:
+                    gr.Info('Checking integrity...', duration=15)
+                    self.update_leaderboard()
+                    
+                    with self.var_lock.ro:
+                        # Is every `submission_id` in results known?
+                        if self.tournament_results.keys() - self.submission_ids != set():
+                            pass
+                        # Was every `submission_id` in some match?
+                        elif self.submission_ids - self.tournament_results.keys() != set():
+                            pass
+                        # Are all competitors known?
+                        elif any(
+                            self.tournament_results[submission_id].keys() - self.submission_ids != set()
+                            for submission_id in self.submission_ids
+                        ):
+                            pass
+                        # Has had every `submission_id` match with all competitors?
+                        elif any(
+                            self.submission_ids - self.tournament_results[submission_id].keys() != set()
+                            for submission_id in self.submission_ids
+                        ):
+                            pass
+                        else:
+                            break
+
+                    gr.Info('Running tournament...', duration=15)
+                    with self.var_lock.rw:
+                        self.tournament_results = {}
+                    for submission_id in self.submission_ids:
+                        with self.var_lock.ro:
+                            file = self.submission_id_to_file[submission_id]
+                            tournament_results = self.start_tournament(submission_id, file)
+                        with self.var_lock.rw:
+                            self.tournament_results = tournament_results
+                    
+                    gr.Info('Uploading tournament results...', duration=5)
+                    if self.tournament_results:
+                        self._upload_tournament_results(self.tournament_results)
+                    
+                    break
+            gr.Info("Waiting in queue...", duration=5)
+            time.sleep(10)
+        
+        gr.Info('Integrity of the results dataset is checked', duration=5)
 
     @staticmethod
     def _model_tournament_table_highlight_true_and_false(x):
@@ -237,11 +298,7 @@ class LeaderboardServer:
         
         with self.var_lock.ro:
             for competitor_id in self.tournament_results[submission_id].keys() - {submission_id}: # without self
-                path = self.submission_id_to_file.get(competitor_id)
-                if path:
-                    data = json.load(open(path))
-                else:
-                    raise gr.Error(f"Submission [{competitor_id}] not found")
+                data = self.submission_id_to_data[competitor_id]
                 
                 match_results = {}
                 for task in self.tournament_results[submission_id][competitor_id]:
@@ -300,16 +357,13 @@ class LeaderboardServer:
             else:
                 processed_results = []
                 for submission_id in tournament_results.keys():
-                    path = self.submission_id_to_file.get(submission_id)
-                    if path is None:
+                    if submission_id not in self.submission_id_to_data:
                         if pre_submit and submission_id == pre_submit.submission_id:
                             data = json.load(open(pre_submit.file))
                         else:
                             raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
-                    elif path:
-                        data = json.load(open(path))
                     else:
-                        raise gr.Error(f"Submission [{submission_id}] not found")
+                        data = self.submission_id_to_data[submission_id]
                     
                     if submission_id != data["metadata"]["submission_id"]:
                         raise gr.Error(f"Proper submission [{submission_id}] not found")
@@ -532,33 +586,38 @@ class LeaderboardServer:
             if self.pre_submit:
                 tournament_results, submission_id, file = self.pre_submit
                 
-                api.upload_file(
-                    path_or_fileobj=file,
-                    path_in_repo=f"data/{submission_id}.json",
-                    repo_id=self.server_address,
-                    repo_type=self.repo_type,
-                    token=HF_TOKEN,
-                )
-
-                # Temporary save tournament results
-                tournament_results_path = os.path.join(self.local_leaderboard, "tournament.json")
-                with open(tournament_results_path, "w") as f:
-                    json.dump(tournament_results, f, sort_keys=True, indent=2)  # readable JSON
-
-                api.upload_file(
-                    path_or_fileobj=tournament_results_path,
-                    path_in_repo="tournament.json",
-                    repo_id=self.server_address,
-                    repo_type=self.repo_type,
-                    token=HF_TOKEN,
-                )
+                self._upload_submission(submission_id, file)
+                self._upload_tournament_results(tournament_results)
                 
                 self.pre_submit = None
 
+    def _upload_submission(self, submission_id, file):
+        api.upload_file(
+            path_or_fileobj=file,
+            path_in_repo=f"data/{submission_id}.json",
+            repo_id=self.server_address,
+            repo_type=self.repo_type,
+            token=HF_TOKEN,
+        )
+
+    def _upload_tournament_results(self, tournament_results):
+        # Temporary save tournament results
+        tournament_results_path = os.path.join(self.local_leaderboard, "tournament.json")
+        with open(tournament_results_path, "w") as f:
+            json.dump(tournament_results, f, sort_keys=True, indent=2)  # readable JSON
+
+        api.upload_file(
+            path_or_fileobj=tournament_results_path,
+            path_in_repo="tournament.json",
+            repo_id=self.server_address,
+            repo_type=self.repo_type,
+            token=HF_TOKEN,
+        )
+
     def get_model_detail(self, submission_id):
-        with self.var_lock.ro:
-            path = self.submission_id_to_file.get(submission_id)
-        if path is None:
+        if submission_id not in self.submission_id_to_data:
             raise gr.Error(f"Submission [{submission_id}] not found")
-        data = json.load(open(path))
-        return data["metadata"]
+        else:
+            with self.var_lock.ro:
+                data = self.submission_id_to_data[submission_id]
+            return data["metadata"]