Cost information and model-model comparison

app.py

@@ -80,31 +80,43 @@ def refresh_leaderboard(
     tossup_df = fetch_tossup_leaderboard(split, style, date, username)
     bonus_df = fetch_bonus_leaderboard(split, style, date, username)
     overall_df = fetch_overall_leaderboard(split, style, date, username)
+
     return tossup_df, bonus_df, overall_df
 
 
 def create_leaderboard_interface(app, refresh_btn, split: str = "tiny_eval", date: datetime.date = None):
     leaderboard_timer = gr.Timer(LEADERBOARD_REFRESH_INTERVAL)
 
-    tossup_df, bonus_df, overall_df = refresh_leaderboard(split, style=False, date=date)
+    tossup_df, bonus_df, overall_df = refresh_leaderboard(split, style=True, date=date)
 
+    gr.HTML(
+        "<div style='font-size: 18px;'>"
+        "ℹ️ <b>E [Score]</b> is the <b>Expected Score</b> for a question. 🙋🏻 and 🤖 indicate the scores against just the Human and the AI players respectively.<br>"
+        "ℹ️ <b>Cost</b> is the cost in USD of executing the pipeline <b>per question prefix</b>. (Typically we have upto ~20 prefixes per tossup question)"
+        "ℹ️ <b>When does the cost matter?</b> When two models buzz at the same token, which they often do, a lighter (cost-effective) model takes precedence.<br>"
+        "</div>"
+    )
     tossup_leaderboard = gr.Dataframe(
         value=tossup_df,
         show_search=True,
         label=" 🛎️ Tossup Round Leaderboard",
         show_label=True,
-        datatype=["str", "number", "number", "number", "number"],
+        datatype=["str", "number", "number", "number", "number", "number", "number"],
         elem_id="tossup-table",
         interactive=False,  # Ensure it's not interactive
     )
 
-    logger.info(f"Bonus dataframe columns: {bonus_df.columns}")
+    gr.HTML(
+        "<div style='font-size: 18px;'>"
+        "ℹ️ <b>Cost for Bonus pipeline</b> is the cost in USD of executing the pipeline <b>per bonus part</b>. (We have exactly 3 parts per bonus question)"
+        "</div>"
+    )
     bonus_leaderboard = gr.Dataframe(
         value=bonus_df,
         show_search=True,
         label=" 🧐 Bonus Round Leaderboard",
         show_label=True,
-        datatype=["str", "number", "number", "number", "number", "number", "number"],
+        datatype=["str", "number", "number", "number", "number", "number", "number", "number", "number"],
         elem_id="bonus-table",
         interactive=False,  # Ensure it's not interactive
     )
@@ -114,7 +126,7 @@ def create_leaderboard_interface(app, refresh_btn, split: str = "tiny_eval", dat
         show_search=True,
         label=" 🥇 Overall Leaderboard",
         show_label=True,
-        datatype=["str", "str", "str", "number", "number", "number", "number", "number"],
+        datatype=["str", "str", "str", "number", "number", "number", "number", "number", "number"],
     )
 
     gr.on(

src/display/css_html_js.py

@@ -102,6 +102,11 @@ table th:first-child {
 #box-filter > .form{
     border: 0
 }
+
+span.multiline.text[role="button"] {
+    font-size: 16px !important;
+}
+
 """
 
 get_window_url_params = """

src/envs.py

@@ -19,7 +19,7 @@ USERS_REPO = f"{OWNER}/registered-users"
 
 ADMIN_USERS = ["mgor"]
 EVAL_SPLITS = {"Week 2": "w2_eval", "Week 1": "w1_eval", "Week 0": "tiny_eval"}
-CUTOFF_DATES = {"Week 1": "2025-05-30", "Week 0": "2025-05-23", "Week 2": "2025-06-07"}
+CUTOFF_DATES = {"Week 1": "2025-05-30", "Week 0": "2025-05-23", "Week 2": "2025-06-10"}
 
 
 # Important Links

src/populate.py

@@ -18,6 +18,9 @@ def fetch_model_results(repo_dir: str, competition_type: str, eval_split: str) -
         if len(files) == 0 or not all(f.endswith(".json") for f in files):
             continue
         for file in files:
+            # Check if the file name is a valid submission id
+            if not file.startswith(f"{competition_type}__"):
+                continue
             filepath = os.path.join(root, file)
             try:
                 with open(filepath, "r") as fp:
@@ -30,6 +33,15 @@ def fetch_model_results(repo_dir: str, competition_type: str, eval_split: str) -
     return model_results
 
 
+def fetch_tossup_elo_results(repo_dir: str, eval_split: str) -> list[dict]:
+    elo_results = []
+    dirpath = os.path.join(repo_dir, "tossup", eval_split)
+    filepath = os.path.join(dirpath, "elo_results.json")
+    with open(filepath, "r") as fp:
+        elo_results = json.load(fp)
+    return elo_results
+
+
 def get_submission_date(result: dict) -> datetime.date:
     submission_id = result["id"]
     datetime_str = submission_id.split("__")[-3]
@@ -52,10 +64,12 @@ def get_tossups_leaderboard_df(
     repo_dir: str, eval_split: str, cutoff_date: datetime.date = None, logged_in_username: str = None
 ) -> pd.DataFrame:
     model_results = fetch_model_results(repo_dir, "tossup", eval_split)
+    elo_results = fetch_tossup_elo_results(repo_dir, eval_split)
 
     eval_results = []
     for result in model_results:
         try:
+            submission_id = result["id"]
             metrics = result["metrics"]
             username = result["username"]
             model_name = result["model_name"]
@@ -64,32 +78,26 @@ def get_tossups_leaderboard_df(
                 if not qualify_for_private_observation(username, logged_in_username):
                     continue
                 submission_name = f"{username}/{model_name} (*)"
-
+            e_score_ai = elo_results.get(submission_id, 0.0)
+            overall_expected_score = 0.5 * (metrics["expected_score"] + e_score_ai)
             row = {
                 "Submission": submission_name,
-                "Expected Score ⬆️": metrics["expected_score"],
-                "Buzz Precision": metrics["buzz_accuracy"],
-                "Buzz Frequency": metrics["buzz_frequency"],
+                "E [Score] ⬆️": overall_expected_score,
+                "E [Score] (🙋🏻)": metrics["expected_score"],
+                "E [Score] (🤖)": e_score_ai,
+                "Cost ⬇️": result["cost"],
+                "Buz Prec.": metrics["buzz_accuracy"],
+                "Buz Freq.": metrics["buzz_frequency"],
                 "Buzz Position": metrics["buzz_position"],
-                "Win Rate w/ Humans": metrics.get("human_win_rate", None),
+                "Win Rate w/ 🙋🏻": metrics.get("human_win_rate", None),
             }
             eval_results.append(row)
         except Exception as e:
-            logger.error(f"Error processing model result '{username}/{model_name}': {e}")
+            logger.error(f"Error processing model result for eval_split={eval_split} '{username}/{model_name}': {e}")
             continue
 
-    df = pd.DataFrame(
-        eval_results,
-        columns=[
-            "Submission",
-            "Expected Score ⬆️",
-            "Buzz Precision",
-            "Buzz Frequency",
-            "Buzz Position",
-            "Win Rate w/ Humans",
-        ],
-    )
-    df.sort_values(by="Expected Score ⬆️", ascending=False, inplace=True)
+    df = pd.DataFrame(eval_results)
+    df.sort_values(by="E [Score] ⬆️", ascending=False, inplace=True)
     return df
 
 
@@ -112,6 +120,7 @@ def get_bonuses_leaderboard_df(
 
             row = {
                 "Submission": submission_name,
+                "Cost ⬇️": result["cost"],
                 "Effect ⬆️": metrics["effectiveness"],
                 "Part Acc": metrics["part_accuracy"],
                 "Question Acc": metrics["question_accuracy"],
@@ -120,13 +129,10 @@ def get_bonuses_leaderboard_df(
             }
             eval_results.append(row)
         except Exception as e:
-            logger.error(f"Error processing model result '{username}/{model_name}': {e}")
+            logger.exception(f"Error processing model result '{username}/{model_name}': {e}")
             continue
 
-    df = pd.DataFrame(
-        eval_results,
-        columns=["Submission", "Effect ⬆️", "Part Acc", "Question Acc", "Calibration", "Adoption"],
-    )
+    df = pd.DataFrame(eval_results)
     df.sort_values(by=["Effect ⬆️", "Question Acc", "Part Acc"], ascending=False, inplace=True)
     return df
 
@@ -135,7 +141,24 @@ def colour_pos_neg(v):
     """Return a CSS rule for the cell that called the function."""
     if pd.isna(v):  # keep NaNs unstyled
         return ""
-    return "color: green;" if v > 0 else "color: red;"
+    return "color: green;" if float(v) > 0 else "color: red;"
+
+
+def color_cost(v):
+    if pd.isna(v):
+        return ""
+    # Bucket the cost into 5 categories with darker colors
+    cost = float(v)
+    if cost < 1:
+        return "color: #006400;"  # dark green
+    elif cost < 2:
+        return "color: #00008b;"  # dark blue
+    elif cost < 3:
+        return "color: #8b8b00;"  # dark yellow
+    elif cost < 4:
+        return "color: #8b4500;"  # dark orange
+    else:
+        return "color: #8b0000;"  # dark red
 
 
 # Helper function to bold the highest value in a column
@@ -154,21 +177,22 @@ def fetch_tossup_leaderboard(
     df = get_tossups_leaderboard_df(EVAL_RESULTS_PATH, split, date, username)
 
     # Apply formatting and styling
+    percent_cols = ["Buz Prec.", "Buz Freq.", "Win Rate w/ 🙋🏻"]
+    float_cols = ["E [Score] ⬆️", "E [Score] (🙋🏻)", "E [Score] (🤖)", "Buzz Position"]
     styled_df = (
         df.style.format(
             {
-                "Expected Score ⬆️": "{:6.3f}",
-                "Buzz Precision": "{:>6.1%}",
-                "Buzz Position": "{:>6.1f}",
-                "Buzz Frequency": "{:>6.1%}",
-                "Win Rate w/ Humans": "{:>6.1%}",
+                **dict.fromkeys(percent_cols, "{:>6.1%}"),
+                **dict.fromkeys(float_cols, "{:6.3f}"),
+                "Cost ⬇️": "${:,.2f}",
             }
         )
-        .map(colour_pos_neg, subset=["Expected Score ⬆️"])
+        .map(colour_pos_neg, subset=["E [Score] ⬆️", "E [Score] (🤖)", "E [Score] (🙋🏻)"])
+        .map(color_cost, subset=["Cost ⬇️"])
         .apply(highlight_private_row, axis=1)
         .apply(
             bold_max,
-            subset=["Expected Score ⬆️", "Buzz Precision", "Buzz Position", "Win Rate w/ Humans"],
+            subset=[*percent_cols, *float_cols],
             axis=0,
         )
     )
@@ -190,9 +214,11 @@ def fetch_bonus_leaderboard(
                 "Effect ⬆️": "{:6.3f}",
                 "Calibration": "{:>6.1%}",
                 "Adoption": "{:>6.1%}",
+                "Cost ⬇️": "${:,.2f}",
             }
         )
         .map(colour_pos_neg, subset=["Effect ⬆️"])
+        .map(color_cost, subset=["Cost ⬇️"])
         .apply(highlight_private_row, axis=1)
         .apply(
             bold_max,
@@ -220,7 +246,7 @@ def create_overall_leaderboard(tossup_df: pd.DataFrame, bonus_df: pd.DataFrame)
     bonus_df["Username"] = bonus_df["Submission"].apply(extract_username)
 
     # Pick best tossup per user (highest Expected Score ⬆️)
-    tossup_best = tossup_df.sort_values("Expected Score ⬆️", ascending=False).drop_duplicates("Username")
+    tossup_best = tossup_df.sort_values("E [Score] ⬆️", ascending=False).drop_duplicates("Username")
     tossup_best = tossup_best.set_index("Username")
 
     # Pick best bonus per user (highest Effect ⬆️)
@@ -244,11 +270,11 @@ def create_overall_leaderboard(tossup_df: pd.DataFrame, bonus_df: pd.DataFrame)
             "Username": merged.index,
             "Tossup Submission": merged["Submission_tossup"].str.split("/").str[1],
             "Bonus Submission": merged["Submission_bonus"].str.split("/").str[1],
-            "Overall Score ⬆️": merged[["Expected Score ⬆️", "Effect ⬆️"]].fillna(0).sum(axis=1),
-            "Expected Score (Tossup) ⬆️": merged["Expected Score ⬆️"],
-            "Effect (Bonus) ⬆️": merged["Effect ⬆️"],
-            "Part Acc (Bonus)": merged["Part Acc"],
-            "Adoption (Bonus)": merged["Adoption"],
+            "Overall Score ⬆️": merged[["E [Score] ⬆️", "Effect ⬆️"]].fillna(0).sum(axis=1),
+            "Tossup Score ⬆️": merged["E [Score] ⬆️"],
+            "Bonus Effect ⬆️": merged["Effect ⬆️"],
+            "Bonus Part Acc": merged["Part Acc"],
+            "Bonus Adoption": merged["Adoption"],
         }
     )
 
@@ -273,10 +299,10 @@ def fetch_overall_leaderboard(
         overall_df.style.format(
             {
                 "Overall Score ⬆️": "{:6.3f}",
-                "Expected Score (Tossup) ⬆️": "{:6.3f}",
-                "Effect (Bonus) ⬆️": "{:6.3f}",
-                "Part Acc (Bonus)": "{:>6.1%}",
-                "Adoption (Bonus)": "{:>6.1%}",
+                "Tossup Score ⬆️": "{:6.3f}",
+                "Bonus Effect ⬆️": "{:6.3f}",
+                "Bonus Part Acc": "{:>6.1%}",
+                "Bonus Adoption": "{:>6.1%}",
             },
             na_rep="-",
         )
@@ -284,13 +310,7 @@ def fetch_overall_leaderboard(
         .apply(highlight_overall_row, axis=1)
         .apply(
             bold_max,
-            subset=[
-                "Overall Score ⬆️",
-                "Expected Score (Tossup) ⬆️",
-                "Effect (Bonus) ⬆️",
-                "Part Acc (Bonus)",
-                "Adoption (Bonus)",
-            ],
+            subset=["Overall Score ⬆️", "Tossup Score ⬆️", "Bonus Effect ⬆️"],
             axis=0,
         )
     )