Fix update collections and dummy references

app.py

@@ -106,7 +106,7 @@ def init_space(full_init: bool = True):
         benchmark_cols=BENCHMARK_COLS,
         show_incomplete=SHOW_INCOMPLETE_EVALS
     )
-    #update_collections(original_df.copy())
+    update_collections(original_df.copy())
     leaderboard_df = original_df.copy()
     
     plot_df = create_plot_df(create_scores_df(raw_data))
@@ -553,10 +553,10 @@ def update_dynamic_files_wrapper():
     except Exception as e:
         print(f"Error updating dynamic files: {e}")
 
-scheduler = BackgroundScheduler()
+scheduler = BackgroundScheduler(daemon=True)
 scheduler.add_job(restart_space, "interval", seconds=10800, next_run_time=datetime.now() + timedelta(hours=3)) # restarted every 3h
-scheduler.add_job(update_dynamic_files_wrapper, "cron", minute=30) # launched every hour on the hour
-scheduler.add_job(update_collections, "date", args=(original_df.copy(),), run_date=datetime.now() + timedelta(minutes=5))
+scheduler.add_job(update_dynamic_files_wrapper, "interval", seconds=1800, next_run_time=datetime.now() + timedelta(minutes=5)) # launched every 30 minutes
+#scheduler.add_job(update_collections, "interval", args=(original_df.copy(),), seconds=3600, next_run_time=datetime.now() + timedelta(minutes=1))
 scheduler.start()
 
 demo.queue(default_concurrency_limit=40).launch()

external_models_results.json

@@ -244,8 +244,8 @@
     },
     {
         "model": "llama_405b_instruct",
-        "name": "meta-llama/Meta-Llama-3.1-405B-Instruct (Vertex AI)",
-        "link": "https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct",
+        "name": "meta-llama/Llama-3.1-405B-Instruct (Vertex AI)",
+        "link": "https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct",
         "date": "2024-08-20",
         "status": "full",
         "main_language": "English",

src/display/utils.py

@@ -111,7 +111,8 @@ baseline_row = {
     AutoEvalColumn.still_on_hub.name: False,
     AutoEvalColumn.moe.name: False,
     AutoEvalColumn.eval_time.name: 0.0,
-    AutoEvalColumn.main_language.name: "?"
+    AutoEvalColumn.main_language.name: "?",
+    'hf_path': None,
 }
 
 baseline_list = []
@@ -156,6 +157,7 @@ human_baseline_row = {
     AutoEvalColumn.moe.name: False,
     AutoEvalColumn.eval_time.name: 0.0,
     AutoEvalColumn.main_language.name: "?",
+    'hf_path': None,
 }
 
 baseline_list = []
@@ -279,7 +281,7 @@ if os.path.exists('external_models_results.json'):
             model_row[AutoEvalColumn.params.name] = model_data['params']
 
         model_row[AutoEvalColumn.main_language.name] = model_data['main_language']
-        #convert 2025-04-03 to 2025-04-03T00:00:00Z
+        model_row['hf_path'] = None if 'huggingface.co' not in model_data['link'] else model_data['link'].split('huggingface.co/')[1]
         external_rows.append(model_row)
 
         #Create external_eval_results
@@ -356,7 +358,7 @@ if os.path.exists('external_models_results.json'):
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn)]
+COLS = [c.name for c in fields(AutoEvalColumn)] + ['hf_path']
 TYPES = [c.type for c in fields(AutoEvalColumn)]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]

src/leaderboard/filter_models.py

@@ -134,7 +134,7 @@ def flag_models(leaderboard_data: list[dict]):
         if model_data[AutoEvalColumn.flagged.name] == True:
             flag_key = "merged"
         else:
-            flag_key = model_data[AutoEvalColumn.dummy.name]
+            flag_key = model_data['hf_path']
 
         if flag_key in FLAGGED_MODELS:
             issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
@@ -153,7 +153,7 @@ def flag_models(leaderboard_data: list[dict]):
 def remove_forbidden_models(leaderboard_data: list[dict]):
     indices_to_remove = []
     for ix, model in enumerate(leaderboard_data):
-        if model[AutoEvalColumn.dummy.name] in DO_NOT_SUBMIT_MODELS:
+        if model['hf_path'] in DO_NOT_SUBMIT_MODELS:
             indices_to_remove.append(ix)
 
     for ix in reversed(indices_to_remove):

src/leaderboard/read_evals.py

@@ -182,9 +182,10 @@ class EvalResult:
             npm.append((res-task.value.baseline)*100.0 / (100.0-task.value.baseline))
         average = round(sum(average)/len(average), 2)
         npm = round(sum(npm)/len(npm), 2)
-
+        
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
+            "hf_path": self.full_model,
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,

src/scripts/update_all_request_files.py

@@ -84,6 +84,7 @@ def update_models(file_path, models, original_leaderboard_files=None):
 def update_dynamic_files():
     """ This will only update metadata for models already linked in the repo, not add missing ones.
     """
+    print("update_dynamic_files running...")
     snapshot_download(
         repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
     )

src/tools/collections.py

@@ -5,6 +5,7 @@ from huggingface_hub import add_collection_item, delete_collection_item, get_col
 from huggingface_hub.utils import HfHubHTTPError
 from pandas import DataFrame
 import numpy as np
+import traceback
 
 from src.display.utils import AutoEvalColumn, ModelType, NUMERIC_INTERVALS
 from src.envs import H4_TOKEN, PATH_TO_COLLECTION
@@ -26,6 +27,7 @@ def update_collections(df: DataFrame):
     """This function updates the Open LLM Leaderboard model collection with the latest best models for
     each size category and type.
     """
+    print("Updating collections...")
     collection = get_collection(collection_slug=PATH_TO_COLLECTION, token=H4_TOKEN)
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
 
@@ -41,17 +43,17 @@ def update_collections(df: DataFrame):
                 collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=H4_TOKEN
             )
         except HfHubHTTPError:
+            traceback.print_exc()
             continue
 
     #filter quantized models
-    df = df[df[AutoEvalColumn.precision.name].isin(['bfloat16', 'float16'])]
+    #df = df[df[AutoEvalColumn.precision.name].isin(['bfloat16', 'float16', "?"])]
     
     ix = 0
+    interval_scores = []
+    interval_itens_languages = []
+    interval_itens = []
     for size in intervals:
-        interval_scores = []
-        interval_itens_languages = []
-        interval_itens = []
-
         numeric_interval = pd.IntervalIndex([intervals[size]])
         mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
         size_df = df.loc[mask]
@@ -71,6 +73,10 @@ def update_collections(df: DataFrame):
             # We add them one by one to the leaderboard
             for i, row in best_models.iterrows():
                 model = row[AutoEvalColumn.dummy.name]
+                hf_path = row['hf_path']
+                hf_path = hf_path if 'meta-llama/Meta-' not in hf_path else hf_path.replace("meta-llama/Meta-", "meta-llama/")
+                if hf_path in cur_best_models:
+                    continue
                 score = row[AutoEvalColumn.average.name]
                 language = row[AutoEvalColumn.main_language.name]
                 if language == 'Portuguese':
@@ -80,7 +86,7 @@ def update_collections(df: DataFrame):
                 try:
                     collection = add_collection_item(
                         PATH_TO_COLLECTION,
-                        item_id=model,
+                        item_id=hf_path,
                         item_type="model",
                         exists_ok=True,
                         note=note,
@@ -88,13 +94,14 @@ def update_collections(df: DataFrame):
                     )
                     ix += 1
                     item_object_id = collection.items[-1].item_object_id
-                    cur_best_models.append(model)
+                    cur_best_models.append(hf_path)
                     interval_scores.append(float(score))
                     interval_itens_languages.append(language)
                     interval_itens.append(item_object_id)
                     scores_per_type[model_type] = float(score)
                     break
                 except HfHubHTTPError:
+                    traceback.print_exc()
                     continue
         if 'Portuguese' not in interval_itens_languages:
             language = ['Portuguese']
@@ -107,6 +114,10 @@ def update_collections(df: DataFrame):
             # We add them one by one to the leaderboard
             for i, row in best_models.iterrows():
                 model = row[AutoEvalColumn.dummy.name]
+                hf_path = row['hf_path']
+                hf_path = hf_path if 'meta-llama/Meta-' not in hf_path else hf_path.replace("meta-llama/Meta-", "meta-llama/")
+                if hf_path in cur_best_models:
+                    continue
                 score = row[AutoEvalColumn.average.name]
                 language = row[AutoEvalColumn.main_language.name]
                 
@@ -117,7 +128,7 @@ def update_collections(df: DataFrame):
                 try:
                     collection = add_collection_item(
                         PATH_TO_COLLECTION,
-                        item_id=model,
+                        item_id=hf_path,
                         item_type="model",
                         exists_ok=True,
                         note=note,
@@ -125,28 +136,31 @@ def update_collections(df: DataFrame):
                     )
                     ix += 1
                     item_object_id = collection.items[-1].item_object_id
-                    cur_best_models.append(model)
+                    cur_best_models.append(hf_path)
                     interval_scores.append(float(score))
                     interval_itens_languages.append(language)
                     interval_itens.append(item_object_id)
                     scores_per_type[model_type] = float(score)
                     break
                 except HfHubHTTPError:
+                    traceback.print_exc()
                     continue
-        # fix order:
-        starting_idx = len(cur_best_models)
-        k = 0
-        for i in np.argsort(interval_scores):
-            if i == k:
-                continue
-            else:
-                try:
-                    update_collection_item(
-                        collection_slug=PATH_TO_COLLECTION, item_object_id=interval_itens[i], position=starting_idx+k
-                    )
-                except:
-                    pass
-            k += 1
+    # fix order:
+    starting_idx = len(cur_best_models)
+    k = 0
+    for i in np.argsort(interval_scores):
+        if i == k:
+            continue
+        else:
+            try:
+                #print(cur_best_models[i], interval_itens[i], starting_idx+k, interval_scores[i])
+                update_collection_item(
+                    collection_slug=PATH_TO_COLLECTION, item_object_id=interval_itens[i], position=starting_idx+k
+                )
+            except:
+                traceback.print_exc()
+                pass
+        k += 1
 
     collection = get_collection(PATH_TO_COLLECTION, token=H4_TOKEN)
     for item in collection.items:
@@ -156,4 +170,5 @@ def update_collections(df: DataFrame):
                     collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=H4_TOKEN
                 )
             except HfHubHTTPError:
+                traceback.print_exc()
                 continue