Add code for CLD

about.py

@@ -15,7 +15,7 @@ STANDARD_COLS = ["Endpoint", "user", "submission_time", "model_report"]
 METRICS = ["MAE", "RAE", "R2", "Spearman R", "Kendall's Tau"]
 # Final columns
 LB_COLS = ["user", "MAE", "R2", "Spearman R", "Kendall's Tau", "submission time", "model details"]
-LB_AVG = ["user", "MA-RAE", "R2", "Spearman R", "Kendall's Tau", "submission time", "model details"] # Delete some columns for overall LB?
+LB_AVG = ["rank", "user", "MA-RAE", "R2", "Spearman R", "Kendall's Tau", "submission time", "model details"] # Delete some columns for overall LB?
 LB_DTYPES = ['markdown', 'number', 'number', 'number', 'number', 'str', 'markdown', 'number']
 
 # Dictionary with unit conversion multipliers for each endpoint

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 from gradio_leaderboard import Leaderboard
 from gradio.themes.utils import sizes
 import pandas as pd
+import numpy as np
 
 from evaluate import submit_data, evaluate_data
 from utils import (
@@ -10,6 +11,7 @@ from utils import (
     fetch_dataset_df,
     map_metric_to_stats,
 )
+from cld import add_cld_to_leaderboard
 from datasets import load_dataset
 import tempfile
 from loguru import logger
@@ -21,7 +23,7 @@ import threading
 
 ALL_EPS = ['Average'] + ENDPOINTS
 
-def build_leaderboard(df_results):
+def build_leaderboard(df_results, df_results_raw):
     logger.info("Rebuilding leaderboard data...")
     per_ep = {}
     for ep in ALL_EPS:
@@ -32,10 +34,7 @@ def build_leaderboard(df_results):
             per_ep[ep] = pd.DataFrame(columns=LB_COLS) # Empty df
             continue
 
-        # Make user and model details clickable if it's a huggingface user
-        df['user'] = df.apply(
-            lambda row: make_user_clickable(row['user']) if not row['anonymous'] else row['user'], 
-            axis=1).astype(str)
+        # Make model details clickable if it's a huggingface user
         df['model details'] = df['model_report'].apply(lambda x: make_tag_clickable(x)).astype(str)
 
         if ep == "Average":
@@ -44,16 +43,38 @@ def build_leaderboard(df_results):
                                     "std_RAE": "std_MA-RAE"}) 
             sorted_df = df.sort_values(by='mean_MA-RAE', ascending=True, kind="stable")
             sorted_df = map_metric_to_stats(sorted_df, average=True)
-            per_ep[ep] = sorted_df[LB_AVG]
+            # Add ranking column
+            sorted_df['rank'] = np.arange(1, len(sorted_df) + 1)
+            avg_leaderboard = sorted_df.copy()
+            avg_cols = LB_AVG
+            # Add CLD
+            if df_results_raw is not None:
+                df_raw = df_results_raw[df_results_raw["Endpoint"] == ep].copy()
+                df_raw = df_raw.rename(columns={"RAE": "MA-RAE"})
+                avg_leaderboard = add_cld_to_leaderboard(
+                    sorted_df,
+                    df_raw,
+                    "MA-RAE",
+                )
+                avg_cols = ["rank", "user", "CLD", "MA-RAE", "R2", "Spearman R", "Kendall's Tau", "submission time", "model details"]
+
+            # Make user and model details clickable if it's a huggingface user
+            avg_leaderboard['user'] = avg_leaderboard.apply(
+            lambda row: make_user_clickable(row['user']) if not row['anonymous'] else row['user'], 
+            axis=1).astype(str)
+            per_ep[ep] = avg_leaderboard[avg_cols]
         else:
             sorted_df = df.sort_values(by="mean_MAE", ascending=True, kind="stable")
             sorted_df = map_metric_to_stats(sorted_df)
+            sorted_df['user'] = sorted_df.apply(
+                lambda row: make_user_clickable(row['user']) if not row['anonymous'] else row['user'], 
+                axis=1).astype(str)
             per_ep[ep] = sorted_df[LB_COLS]
     logger.info("Finished rebuilding leaderboard data.")
     return per_ep
     
 # Initialize global dataframe
-current_df = fetch_dataset_df()
+current_df, current_df_raw = fetch_dataset_df()
 
 # # Initialize global counter
 # data_version_counter = 0
@@ -64,9 +85,9 @@ def update_current_dataframe():
     global current_df # ugly but works
     while True:
         logger.info("Fetching latest dataset for leaderboard...")
-        current_df = fetch_dataset_df()
+        current_df, current_df_raw = fetch_dataset_df()
         logger.debug(f"Dataset version updated")
-        time.sleep(60)  # Check for updates every 60 seconds
+        time.sleep(300)  # Check for updates every 5 minutes
 
 threading.Thread(target=update_current_dataframe, daemon=True).start()
 
@@ -174,7 +195,7 @@ with gr.Blocks(title="OpenADMET ADMET Challenge", fill_height=False,
 
     The test set will remained blinded until the challenge submission deadline. You will be tasked with predicting the same set of ADMET endpoints for the test set molecules.
     
-    The training and blinded test set will also be made available on the [CDD Vault](https://www.collaborativedrug.com/). An account to access the CDD Vault can be requested by filling out this [form](https://forms.gle/KiviZ7AaGcuqtrwH8, which can also be used to request access to some other tools.
+    The training and blinded test set will also be made available on the [CDD Vault](https://www.collaborativedrug.com/). An account to access the CDD Vault can be requested by filling out this [form](https://forms.gle/KiviZ7AaGcuqtrwH8), which can also be used to request access to some other tools.
     Note that by joining the Vault, your account will be visible to other participants, so this option is **not recommended for those wishing to remain anonymous.**
 
     ## 📝 Evaluation
@@ -251,28 +272,28 @@ with gr.Blocks(title="OpenADMET ADMET Challenge", fill_height=False,
             # Aggregated leaderboard
             with gr.TabItem('OVERALL', elem_id="all_tab"):
                 lboard_dict['Average'] = Leaderboard(
-                    value=build_leaderboard(current_df)['Average'],
-                    datatype=LB_DTYPES,
+                    value=build_leaderboard(current_df, current_df_raw)['Average'],
+                    datatype=['number'] + LB_DTYPES,
                     select_columns=LB_AVG,
                     search_columns=["user"],
                     render=True,
-                    every=30,
+                    every=300,
                 )
             # per-endpoint leaderboard
             for endpoint in ENDPOINTS:
                 with gr.TabItem(endpoint):
                     lboard_dict[endpoint] = Leaderboard(
-                        value=build_leaderboard(current_df)[endpoint],
+                        value=build_leaderboard(current_df, current_df_raw)[endpoint],
                         datatype=LB_DTYPES,
                         select_columns=LB_COLS,
                         search_columns=["user"],
                         render=True,
-                        every=30,
+                        every=300,
                     )
             # Auto-refresh 
             def refresh_if_changed():
                 logger.info("Refreshing on timer tick...")
-                per_ep = build_leaderboard(current_df)
+                per_ep = build_leaderboard(current_df, current_df_raw)
                 #return [gr.update(value=per_ep.get(ep, pd.DataFrame(columns=LB_COLS))) for ep in ALL_EPS]
                 return [per_ep[ep] for ep in ALL_EPS]
             data_version.change(fn=refresh_if_changed, outputs=[lboard_dict[ep] for ep in ALL_EPS])

cld.py

@@ -0,0 +1,204 @@
+from statsmodels.stats.multicomp import pairwise_tukeyhsd
+from string import ascii_lowercase, ascii_uppercase
+import tqdm
+import pandas as pd
+
+CLD_ALPHABET = list(ascii_lowercase) + list(ascii_uppercase)
+
+def asserts_non_significance(col: list[bool], i: int, j: int) -> bool:
+    """Assert whether i and j are represented as non-significant in the column 
+    i.e., if the corresponding values in the column are different
+
+    Parameters
+    ----------
+    col : list[bool]
+        current column
+    i : int
+        index of first treatment
+    j : int
+        index of second treatment
+
+    Returns
+    -------
+    bool
+        If the non-significance is represented accurately
+    """
+    return col[i] and col[j]
+
+def insert(column: list[bool], i: int, j: int):
+    """Duplicates column and in one of its copies flip entry i to 0, 
+    and in the other copy flip entry j to 0
+
+    Parameters
+    ----------
+    column : list[bool]
+        Original column
+    i : int
+       Index of first group
+    j : int
+       Index of second group
+
+    Returns
+    -------
+    list[bool], list[bool]
+       New columns after duplication and flip
+    """
+    col_i = column.copy()
+    col_j = column.copy()
+    col_i[i] = False
+    col_j[j] = False
+    return col_i, col_j
+
+def can_be_absorbed(new_col: list[bool], ref_col: list[bool]) -> bool: 
+    """An old column absorbs the new column 
+    if it has a 1 in every row in which the new column has one
+
+    Parameters
+    ----------
+    new_col : list[bool]
+        Column to add
+    ref_col : list[bool]
+        Old column we are checking if it can absorb new_col
+
+    Returns
+    -------
+    bool
+       Whether old column cand absorb new_col
+    """
+    return all(ref_col[i] for i, x in enumerate(new_col) if x)
+
+def absorb(new_column: list[bool], columns: list[list[bool]]) -> list[list[bool]]:
+    """Absorb new column into existing columns if the condition allows
+
+    Parameters
+    ----------
+    new_column : list[bool]
+        Column to add
+    columns : list[list[bool]]
+        existing columns
+
+    Returns
+    -------
+    list[list[bool]]
+        Columns after absorption
+    """
+    if any(can_be_absorbed(new_column, c) for c in columns):
+        return columns
+    return columns + [new_column]
+
+def cld(comparisons: pd.DataFrame) -> dict[str, str]:
+    """
+    Compact Letter Display
+
+    Compute the compact letter display using the insert-absorb algorithm.
+
+    See the following papers for more information:
+    (1) https://doi.org/10.1016/j.csda.2006.09.035
+    (2) https://doi.org/10.1198/1061860043515
+
+    Parameters
+    ----------
+        comparisons : pd.DataFrame
+            A DataFrame containing the pairwise comparisons produced by:
+            https://www.statsmodels.org/dev/generated/statsmodels.stats.multicomp.pairwise_tukeyhsd.html
+    """
+    unique_groups = set(comparisons["group1"].unique())
+    unique_groups = unique_groups.union(set(comparisons["group2"].unique()))
+    unique_groups = list(unique_groups)
+    unique_groups_indices = {g: i for i, g in enumerate(unique_groups)}
+
+    sig_diff = comparisons[comparisons["reject"]]
+    print(f"Found {len(sig_diff)} significantly different pairs")
+
+    # Initialize CLD matrix for all unique groups/models, with "columns" as rows
+    solution = [[True] * len(unique_groups)]
+
+    for _, row in tqdm.tqdm(sig_diff.iterrows(), total=len(sig_diff)):
+        i = unique_groups_indices[row["group1"]]
+        j = unique_groups_indices[row["group2"]]
+
+        has_changed: bool = True
+        while has_changed:
+            has_changed = False
+
+            for idx in range(len(solution)):
+                if asserts_non_significance(solution[idx], i, j):
+                    # Duplicate the column
+                    col_i, col_j = insert(solution[idx], i, j)
+
+                    # Remove the old column
+                    solution.pop(idx)
+
+                    # Try absorb the column in an old column
+                    # Simply add it to the solution otherwise
+                    solution = absorb(col_i, solution)
+                    solution = absorb(col_j, solution)
+
+                    has_changed = True
+                    break
+
+    # Assign letters
+    letters = [""] * len(unique_groups)
+
+    for ci, col in enumerate(solution):
+        letter = CLD_ALPHABET[ci]
+        for idx, has_letter in enumerate(col):
+            if has_letter:
+                letters[idx] += letter
+
+    return {group: sorted(letter) for group, letter in zip(unique_groups, letters)}
+
+from statsmodels.stats.multicomp import pairwise_tukeyhsd
+import tqdm
+
+def add_cld_to_leaderboard(
+    leaderboard: pd.DataFrame,
+    scores: pd.DataFrame,
+    metric: str,
+):
+    """Add the compact letter display to the leaderboard.
+
+    Parameters
+    ----------
+    leaderboard : pd.DataFrame
+        The full leaderboard DataFrame
+    scores : pd.DataFrame
+        The **raw** scores DataFrame, with all replicates from bootstrapping
+    metric_ : str
+        The metric label to calculate CLD for.
+    """
+    ordered_methods = leaderboard["user"].values
+
+    scores = scores[["Sample", "user", metric]]
+    scores[metric] = scores[metric].astype(float)
+
+    # We compared methods using bootstrapping and the Tukey HSD test, presenting results via Compact Letter Display (CLD).
+    # While acknowledging that bootstrapping likely underestimates variance,
+    # we are not aware of better sampling techniques that fit the challenge format.
+    stats = pairwise_tukeyhsd(endog=scores[metric], groups=scores["user"])
+    # comparisons = stats.summary_frame()
+    # The version of statsmodel is for some reason not the latest, so we have to do small workaround to get summary_frame
+    summary_table = stats.summary()
+    # data attribute is a list of lists with column names as first element
+    data = summary_table.data[1:]
+    columns = summary_table.data[0]
+    comparisons = pd.DataFrame(data=data, columns=columns)
+
+    letter_mapping = {}
+    letter_code = cld(comparisons)
+
+    cld_column = [""] * len(leaderboard)
+    for idx, method in enumerate(ordered_methods):
+        try:
+            letters = letter_code[str(method)]
+
+            for letter in letters:
+                if letter not in letter_mapping:
+                    letter_mapping[letter] = CLD_ALPHABET[len(letter_mapping)]
+                cld_column[idx] += letter_mapping[letter]
+        except KeyError: # Error with CLD for openadmet-dummy
+            cld_column[idx] = "None"
+
+    leaderboard["CLD"] = cld_column
+
+    return leaderboard

evaluate.py

@@ -257,6 +257,7 @@ def _evaluate_data(filename: str, test_repo: str, split_filename: str, results_r
     results_df['anonymous'] = meta.participant.anonymous
     results_df['hf_username'] = username
 
+    results_raw_df = results_raw_df[results_raw_df['Endpoint']=='Average'] # Save ONLY for average endpoint, otherwise file is too large
     results_raw_df['user'] = display_name
     results_raw_df['submission_time'] = timestamp
     results_raw_df['model_report'] = report

utils.py

@@ -15,7 +15,7 @@ def make_tag_clickable(tag: str):
         return "Not submitted"
     return f'<a target="_blank" href="{tag}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">link</a>'
 
-def fetch_dataset_df():
+def fetch_dataset_df(download_raw=False): # Change download_raw to True for the final leaderboard
     logger.info("Fetching latest results dataset from Hugging Face Hub...")
     # Specify feature types to load results dataset
     metric_features = {
@@ -60,7 +60,34 @@ def fetch_dataset_df():
           .reset_index(drop=True)
     )
     latest.rename(columns={"submission_time": "submission time"}, inplace=True)
-    return latest
+
+    # Also fetch raw dataset
+    metric_features = {
+        m: Value('float64') for m in METRICS
+    }
+    other_features.update({'Sample': Value("float32")})
+    feature_schema = Features(metric_features | other_features)
+
+    # We'll set download_raw for the live leaderboard, as it too long to load
+    latest_raw = None
+    if download_raw:
+        dset_raw = load_dataset(results_repo_validation, # change to results_repo_test for test set
+                            name='raw',
+                            split='train', 
+                            features=feature_schema,
+                            download_mode="force_redownload")
+        raw_df = dset_raw.to_pandas()
+        df_raw = raw_df.copy()
+        df_raw["submission_time"] = pd.to_datetime(df_raw["submission_time"], errors="coerce")
+        df_raw = df_raw.dropna(subset=["submission_time"])
+        latest_raw = (
+            df_raw.sort_values("submission_time")
+            .drop_duplicates(subset=["Sample", "Endpoint", "hf_username"], keep="last") 
+            .sort_values(["Sample","Endpoint", "user"])
+            .reset_index(drop=True)
+        )
+
+    return latest, latest_raw
 
 
 def clip_and_log_transform(y: np.ndarray):