leaderboard code v2

cld.py

@@ -4,10 +4,10 @@ import tqdm
 import pandas as pd
 from itertools import product
 
-# Make large CLD alphabet with lowercase, uppercase and double letter combinations
+# Make large CLD alphabet
 single_chars = list(ascii_lowercase) + list(ascii_uppercase) 
-double_chars = [''.join(p) for p in product(single_chars, repeat=2)]
-CLD_ALPHABET = single_chars + double_chars # length is 2756
+underscore_chars = [''.join(p) for p in product(['_'], single_chars)]
+CLD_ALPHABET = single_chars + underscore_chars
 
 def asserts_non_significance(col: list[bool], i: int, j: int) -> bool:
     """Assert whether i and j are represented as non-significant in the column 

final_lb.py

@@ -5,7 +5,7 @@ from utils import (
     map_metric_to_stats,
     fetch_dataset_df,
 )
-from about import ENDPOINTS, LB_COLS, LB_AVG, results_repo_test
+from about import ENDPOINTS, LB_COLS, results_repo_test
 
 from loguru import logger
 import pandas as pd
@@ -35,36 +35,59 @@ def build_leaderboard(df_results, df_results_raw, avg_only=True):
             df['hf_username'] = df['hf_username'].apply(lambda s: s.lower())
             df = df.sort_values(by="submission time", ascending=False, kind="stable")
             df = df.drop_duplicates(subset=['hf_username'], keep='first')
+
             # Sort by MAE-RAE
             sorted_df = df.sort_values(by='mean_MA-RAE', ascending=True, kind="stable")
             sorted_df = map_metric_to_stats(sorted_df, average=True)
-            avg_leaderboard = sorted_df.copy()
-            avg_cols = LB_AVG
-            # Add CLD
+
+            # Make sure Hugging Face username exists, if not, delete the row
+            sorted_df['user_real'] = sorted_df['hf_username'].apply(validate_hf_username)
+            sorted_df_clean = sorted_df[sorted_df['user_real']].reset_index(drop=True)
+            # Add ranking column
+            sorted_df_clean['rank'] = np.arange(1, len(sorted_df_clean) + 1)
+            avg_leaderboard = sorted_df_clean.copy()
+
+            # Clean raw data as well
             df_raw = df_results_raw[df_results_raw["Endpoint"] == ep].copy()
             df_raw = df_raw.rename(columns={"RAE": "MA-RAE"})
+
+            df_raw['hf_username'] = df_raw['hf_username'].apply(lambda s: s.lower())
+            df_raw = df_raw.sort_values(by="submission_time", ascending=False, kind="stable")
+            df_raw = df_raw.drop_duplicates(subset=['hf_username','Sample'], keep='first')
+
+            valid_usernames = sorted_df_clean['hf_username'].unique()
+            df_raw_clean = df_raw[df_raw['hf_username'].isin(valid_usernames)].reset_index(drop=True)
+
+            # Make sure order of raw dataframe is the same as sorted dataframe
+            username_order = sorted_df['hf_username'].unique()
+            df_raw_sorted = df_raw_clean.copy()
+            df_raw_sorted['hf_username'] = pd.Categorical(
+                df_raw_sorted['hf_username'], 
+                categories=username_order, 
+                ordered=True
+            )
+            df_raw_sorted = df_raw_sorted.sort_values(
+                by=['hf_username', 'Sample'], 
+                ascending=[True, True]
+            )
+            df_raw_sorted['hf_username'] = df_raw_sorted['hf_username'].astype(str)
+            df_raw_sorted = df_raw_sorted.reset_index(drop=True)
+
             avg_leaderboard = add_cld_to_leaderboard(
-                sorted_df,
-                df_raw,
+                sorted_df_clean,
+                df_raw_sorted,
                 "MA-RAE",
             )
             avg_cols = ["rank",
-                         "user", 
-                         "CLD", 
-                         "MA-RAE", 
-                         "R2", 
-                         "Spearman R", 
-                         "Kendall's Tau", 
-                         "model details"]
-
-            # Make sure Hugging Face username exists, if not, delete the row
-            avg_leaderboard['user_real'] = avg_leaderboard['hf_username'].apply(validate_hf_username)
-            avg_leaderboard_clean = avg_leaderboard[avg_leaderboard['user_real']]
-
-            # Add ranking column
-            avg_leaderboard_clean['rank'] = np.arange(1, len(avg_leaderboard_clean) + 1)
+                        "user", 
+                        "CLD", 
+                        "MA-RAE", 
+                        "R2", 
+                        "Spearman R", 
+                        "Kendall's Tau", 
+                        "model details"]
 
-            per_ep[ep] = avg_leaderboard_clean[avg_cols]
+            per_ep[ep] = avg_leaderboard[avg_cols]
 
         else:
             if avg_only:
@@ -109,6 +132,8 @@ def prepare_lb_csv(save_folder:str, avg_only:bool):
     per_ep_df = build_leaderboard(df_latest, df_latest_raw, avg_only)
     logger.info("Saving leaderboard")
     for ep in ALL_EPS:
+        if ep != "Average" and avg_only:
+            continue
         df_lb = per_ep_df[ep]
         save_path = Path(save_folder) / f"{ep}_leaderboard.csv"
         df_lb.to_csv(save_path, index=False)