update

app.py

@@ -11,18 +11,24 @@ from src.about import (
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
-    nc_tasks
+    nc_tasks,
+    nr_tasks,
+    lp_tasks,
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
     COLS,
     COLS_NC,
+    COLS_NR,
+    COLS_LP,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
     TYPES,
     AutoEvalColumn_NodeClassification,
+    AutoEvalColumn_NodeRegression,
+    AutoEvalColumn_LinkPrediction,
     #AutoEvalColumn,
     ModelType,
     TASK_LIST,
@@ -56,9 +62,6 @@ except Exception:
     restart_space()
 
 
-original_df = get_leaderboard_df(EVAL_REQUESTS_PATH, nc_tasks)
-leaderboard_df = original_df.copy()
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,
@@ -72,7 +75,7 @@ def update_table(
 
 
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn_NodeClassification.model.name].str.contains(query, case=False))]
+    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
 
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
@@ -81,7 +84,7 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     ]
     # We use COLS to maintain sorting
     filtered_df = df[
-        always_here_cols + [c for c in COLS_NC if c in df.columns and c in columns] 
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
     ]
     return filtered_df
 
@@ -99,7 +102,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
         if len(final_df) > 0:
             filtered_df = pd.concat(final_df)
             filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn_NodeClassification.model.name]
+                subset=[AutoEvalColumn.model.name]
             )
 
     return filtered_df
@@ -112,14 +115,14 @@ def filter_models(
     if show_deleted:
         filtered_df = df
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn_NodeClassification.still_on_hub.name] == True]
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
     #type_emoji = [t[0] for t in type_query]
     #filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     #filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn_NodeClassification.params.name], errors="coerce")
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
     mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
     filtered_df = filtered_df.loc[mask]
 
@@ -132,6 +135,155 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 Node Classification Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
+            COLS = COLS_NC
+            AutoEvalColumn = AutoEvalColumn_NodeClassification
+            original_df = get_leaderboard_df(EVAL_REQUESTS_PATH, "Node Classification")
+            leaderboard_df = original_df.copy()
+
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden
+                            ],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    
+            print(leaderboard_df)
+            print(fields(AutoEvalColumn))
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_df[
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                    + shown_columns.value
+                ],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df[COLS],
+                headers=COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            for selector in [shown_columns]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+        
+        with gr.TabItem("🏅 Node Regression Leaderboard", elem_id="llm-benchmark-tab-table", id=1):
+            COLS = COLS_NR
+            AutoEvalColumn = AutoEvalColumn_NodeRegression
+            original_df = get_leaderboard_df(EVAL_REQUESTS_PATH, "Node Regression")
+            leaderboard_df = original_df.copy()
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden
+                            ],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    
+            print(leaderboard_df)
+            print(fields(AutoEvalColumn))
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_df[
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                    + shown_columns.value
+                ],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df[COLS],
+                headers=COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            for selector in [shown_columns]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
+        with gr.TabItem("🏅 Link Prediction Leaderboard", elem_id="llm-benchmark-tab-table", id=2):
+            COLS = COLS_LP
+            AutoEvalColumn = AutoEvalColumn_LinkPrediction
+            original_df = get_leaderboard_df(EVAL_REQUESTS_PATH, "Link Prediction")
+            leaderboard_df = original_df.copy()
             with gr.Row():
                 with gr.Column():
                     with gr.Row():
@@ -144,12 +296,12 @@ with demo:
                         shown_columns = gr.CheckboxGroup(
                             choices=[
                                 c.name
-                                for c in fields(AutoEvalColumn_NodeClassification)
+                                for c in fields(AutoEvalColumn)
                                 if not c.hidden and not c.never_hidden
                             ],
                             value=[
                                 c.name
-                                for c in fields(AutoEvalColumn_NodeClassification)
+                                for c in fields(AutoEvalColumn)
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden
                             ],
                             label="Select columns to show",
@@ -158,13 +310,13 @@ with demo:
                         )
                     
             print(leaderboard_df)
-            print(fields(AutoEvalColumn_NodeClassification))
+            print(fields(AutoEvalColumn))
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn_NodeClassification) if c.never_hidden]
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                     + shown_columns.value
                 ],
-                headers=[c.name for c in fields(AutoEvalColumn_NodeClassification) if c.never_hidden] + shown_columns.value,
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -173,8 +325,8 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS_NC],
-                headers=COLS_NC,
+                value=original_df[COLS],
+                headers=COLS,
                 datatype=TYPES,
                 visible=False,
             )

src/about.py

@@ -21,17 +21,40 @@ NUM_FEWSHOT = 0 # Change with your few shot
 class nc_tasks(Enum):
     task0 = Task("rel-amazon/user-churn", "auroc", "user-churn")
     task1 = Task("rel-amazon/item-churn", "auroc", "item-churn")
-    task2 = Task("rel-avito/user-clicks", "auroc", "user-clicks")
     task3 = Task("rel-avito/user-visits", "auroc", "user-visits")
-    task4 = Task("rel-hm/user-churn", "auroc", "hm-user-churn")
-    task5 = Task("rel-stack/user-badge", "auroc", "user-badge")
-    task6 = Task("rel-stack/user-engagement", "auroc", "user-engagement")
+    task2 = Task("rel-avito/user-clicks", "auroc", "user-clicks")
     task7 = Task("rel-f1/driver-dnf", "auroc", "driver-dnf")
     task8 = Task("rel-f1/driver-top3", "auroc", "driver-top3")
+    task4 = Task("rel-hm/user-churn", "auroc", "hm-user-churn")
+    task6 = Task("rel-stack/user-engagement", "auroc", "user-engagement")
+    task5 = Task("rel-stack/user-badge", "auroc", "user-badge")
     task9 = Task("rel-trial/study-outcome", "auroc", "study-outcome")
     task10 = Task("rel-event/user-repeat", "auroc", "user-repeat")
     task11 = Task("rel-event/user-ignore", "auroc", "user-ignore")
 
+
+class nr_tasks(Enum):
+    task0 = Task("rel-amazon/user-ltv", "mae", "user-ltv")
+    task1 = Task("rel-amazon/item-ltv", "mae", "item-ltv")
+    task3 = Task("rel-avito/ad-ctr", "mae", "ad-ctr")
+    task4 = Task("rel-f1/driver-position", "mae", "driver-position")
+    task5 = Task("rel-hm/item-sales", "mae", "item-sales")
+    task6 = Task("rel-stack/post-votes", "mae", "post-votes")
+    task7 = Task("rel-trial/study-adverse", "mae", "study-adverse")
+    task8 = Task("rel-trial/site-success", "mae", "site-success")
+    task9 = Task("rel-event/user-attendance", "mae", "user-attendance")
+
+class lp_tasks(Enum):
+    task0 = Task("rel-amazon/user-item-purchase", "map", "user-item-purchase")
+    task1 = Task("rel-amazon/user-item-rate", "map", "user-item-rate")
+    task2 = Task("rel-amazon/user-item-review", "map", "user-item-review")
+    task3 = Task("rel-avito/user-ad-visit", "map", "user-ad-visit")
+    task4 = Task("rel-hm/user-item-purchase", "map", "hm-user-item-purchase")
+    task5 = Task("rel-stack/user-post-comment", "map", "user-post-comment")
+    task6 = Task("rel-stack/post-post-related", "map", "post-post-related")
+    task7 = Task("rel-trial/condition-sponsor-run", "map", "condition-sponsor-run")
+    task8 = Task("rel-trial/site-sponsor-run", "map", "site-sponsor-run")
+
 # Your leaderboard name
 TITLE = """<p align="center"><img src="https://relbench.stanford.edu/img/logo.png" alt="logo" width="400px" /></p>"""
 

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks, nc_tasks
+from src.about import Tasks, nc_tasks, nr_tasks, lp_tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -58,6 +58,37 @@ auto_eval_column_dict_nc.append(["num_of_Params", ColumnContent, ColumnContent("
 
 AutoEvalColumn_NodeClassification = make_dataclass("AutoEvalColumn_NodeClassification", auto_eval_column_dict_nc, frozen=True)
 
+
+auto_eval_column_dict_nr = []
+auto_eval_column_dict_nr.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict_nr.append(["average_rank", ColumnContent, ColumnContent("Average Rank⬆️", "number", True)])
+for task in nr_tasks:
+    auto_eval_column_dict_nr.append(['_'.join(task.value.col_name.split('-')), ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+auto_eval_column_dict_nr.append(["author", ColumnContent, ColumnContent("Author", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nr.append(["email", ColumnContent, ColumnContent("Email", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nr.append(["Paper_URL", ColumnContent, ColumnContent("Paper URL", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nr.append(["Github_URL", ColumnContent, ColumnContent("Github URL", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nr.append(["Time", ColumnContent, ColumnContent("Time", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_nr.append(["num_of_Params", ColumnContent, ColumnContent("# of Params", "markdown", True, never_hidden=False)])
+
+AutoEvalColumn_NodeRegression = make_dataclass("AutoEvalColumn_NodeRegression", auto_eval_column_dict_nr, frozen=True)
+
+
+auto_eval_column_dict_lp = []
+auto_eval_column_dict_lp.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict_lp.append(["average_rank", ColumnContent, ColumnContent("Average Rank⬆️", "number", True)])
+for task in lp_tasks:
+    auto_eval_column_dict_lp.append(['_'.join(task.value.col_name.split('-')), ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+auto_eval_column_dict_lp.append(["author", ColumnContent, ColumnContent("Author", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_lp.append(["email", ColumnContent, ColumnContent("Email", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_lp.append(["Paper_URL", ColumnContent, ColumnContent("Paper URL", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_lp.append(["Github_URL", ColumnContent, ColumnContent("Github URL", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_lp.append(["Time", ColumnContent, ColumnContent("Time", "markdown", True, never_hidden=False)])
+auto_eval_column_dict_lp.append(["num_of_Params", ColumnContent, ColumnContent("# of Params", "markdown", True, never_hidden=False)])
+
+AutoEvalColumn_LinkPrediction = make_dataclass("AutoEvalColumn_LinkPrediction", auto_eval_column_dict_lp, frozen=True)
+
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -182,6 +213,9 @@ class Precision(Enum):
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_NC = [c.name for c in fields(AutoEvalColumn_NodeClassification) if not c.hidden]
+COLS_NR = [c.name for c in fields(AutoEvalColumn_NodeRegression) if not c.hidden]
+COLS_LP = [c.name for c in fields(AutoEvalColumn_LinkPrediction) if not c.hidden]
+
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]

src/populate.py

@@ -6,7 +6,11 @@ import pandas as pd
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
-
+from src.about import (
+    nc_tasks,
+    nr_tasks,
+    lp_tasks,
+)
 
 '''
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -23,7 +27,16 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     return raw_data, df
 '''
 
-def get_leaderboard_df(EVAL_REQUESTS_PATH, tasks) -> pd.DataFrame:
+def get_leaderboard_df(EVAL_REQUESTS_PATH, task_type) -> pd.DataFrame:
+    if task_type == 'Node Classification':
+        ascending = False
+        tasks = nc_tasks
+    elif task_type == 'Node Regression':
+        ascending = True
+        tasks = nr_tasks
+    elif task_type == 'Link Prediction':
+        ascending = False
+        tasks = lp_tasks
 
     model_result_filepaths = []
     for root,_, files in os.walk(EVAL_REQUESTS_PATH):
@@ -36,7 +49,9 @@ def get_leaderboard_df(EVAL_REQUESTS_PATH, tasks) -> pd.DataFrame:
     for model in model_result_filepaths:
         import json
         with open(model) as f:
-            model_res.append(json.load(f))
+            out = json.load(f)
+            if ('task' in out) and (out['task'] == task_type):
+                model_res.append(out)
 
     for model in model_res:
         model["test"] = literal_eval(model["test"])
@@ -61,7 +76,7 @@ def get_leaderboard_df(EVAL_REQUESTS_PATH, tasks) -> pd.DataFrame:
 
     #df_res = pd.DataFrame([{col: model[col] for col in columns_to_show} for model in model_res])
     print(df_res)
-    ranks = df_res[list(name2short_name.values())].rank(ascending = False)
+    ranks = df_res[list(name2short_name.values())].rank(ascending = ascending)
     df_res.rename(columns={'model': 'Model', 'author': 'Author', 'email': 'Email', 'paper_url': 'Paper URL', 'github_url': 'Github URL', 'submitted_time': 'Time', 'params': '# of Params'}, inplace=True)
     df_res['Average Rank⬆️'] = ranks.mean(axis=1)
     df_res.sort_values(by='Average Rank⬆️', ascending=True, inplace=True)

src/submission/submit.py

@@ -44,7 +44,14 @@ def add_new_eval(
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    model_path = model
+    if task_track == 'Node Classification':
+        task_type = 'nc'
+    elif task_track == 'Node Regression':
+        task_type = 'nr'
+    elif task_track == 'Link Prediction':
+        task_type = 'lp'
+
+    model_path = model + '_' + task_type
 
     #precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")