Upload app.py

app.py

@@ -0,0 +1,249 @@
+import gradio as gr
+import pandas as pd
+from huggingface_hub import HfApi
+
+
+DATASETS = [
+    "mMARCO-fr", 
+    "BSARD",
+]
+DENSE_SINGLE_BIENCODERS = [
+    "antoinelouis/biencoder-camembert-base-mmarcoFR",
+    "antoinelouis/biencoder-distilcamembert-mmarcoFR",
+    "antoinelouis/biencoder-mMiniLMv2-L12-mmarcoFR",
+    "antoinelouis/biencoder-camemberta-base-mmarcoFR",
+    "antoinelouis/biencoder-electra-base-french-mmarcoFR",
+    "antoinelouis/biencoder-mMiniLMv2-L6-mmarcoFR",
+    "antoinelouis/biencoder-camembert-L10-mmarcoFR",
+    "antoinelouis/biencoder-camembert-L8-mmarcoFR",
+    "antoinelouis/biencoder-camembert-L6-mmarcoFR",
+    "antoinelouis/biencoder-camembert-L4-mmarcoFR",
+    "antoinelouis/biencoder-camembert-L2-mmarcoFR",
+]
+DENSE_MULTI_BIENCODERS = [
+    "antoinelouis/colbertv1-camembert-base-mmarcoFR",
+    "antoinelouis/colbertv2-camembert-L4-mmarcoFR",
+    "antoinelouis/colbert-xm",
+]
+SPARSE_SINGLE_BIENCODERS = []
+CROSS_ENCODERS = []
+LLMS = []
+COLUMNS = {
+    "Model": "html",
+    "#Params (M)": "number",
+    "Type": "str",
+    "Dataset": "str",
+    "Recall@1000": "number",
+    "Recall@500": "number",
+    "Recall@100": "number",
+    "Recall@10": "number",
+    "MRR@10": "number",
+    "nDCG@10": "number",
+    "MAP@10": "number",
+}
+
+
+def get_model_info(model_id: str, model_type: str) -> pd.DataFrame:
+    data = {}
+    api = HfApi()
+    model_info = api.model_info(model_id)
+    for result in model_info.card_data.eval_results:
+        if result.dataset_name in DATASETS and result.dataset_name not in data:
+            data[result.dataset_name] = {key: None for key in COLUMNS.keys()}
+            data[result.dataset_name]["Model"] = f'<a href="https://huggingface.co/{model_id}" target="_blank" style="color: blue; text-decoration: none;">{model_id}</a>'
+            data[result.dataset_name]["#Params (M)"] = round(model_info.safetensors.total/1e6) if model_info.safetensors else None
+            data[result.dataset_name]["Type"] = model_type
+            data[result.dataset_name]["Dataset"] = result.dataset_name
+        
+        if result.dataset_name in DATASETS and result.metric_name in data[result.dataset_name]:
+            data[result.dataset_name][result.metric_name] = result.metric_value
+    
+    return pd.DataFrame(list(data.values()))
+
+def load_all_results() -> pd.DataFrame:
+    df = pd.DataFrame()
+    for model_id in DENSE_SINGLE_BIENCODERS:
+        df = pd.concat([df, get_model_info(model_id, model_type="DSVBE")])
+    for model_id in DENSE_MULTI_BIENCODERS:
+        df = pd.concat([df, get_model_info(model_id, model_type="DMVBE")])
+    for model_id in SPARSE_SINGLE_BIENCODERS:
+        df = pd.concat([df, get_model_info(model_id, model_type="SSVBE")])
+    for model_id in CROSS_ENCODERS:
+        df = pd.concat([df, get_model_info(model_id, model_type="CE")])
+    for model_id in LLMS:
+        df = pd.concat([df, get_model_info(model_id, model_type="LLM")])
+    return df
+
+def filter_dataf_by_dataset(dataf: pd.DataFrame, dataset_name: str, sort_by: str) -> pd.DataFrame:
+    return (dataf
+        .loc[dataf["Dataset"] == dataset_name]
+        .drop(columns=["Dataset"])
+        .sort_values(by=sort_by, ascending=False)
+    )
+
+
+def update_table(dataf: pd.DataFrame, query: str, selected_types: list, selected_sizes: list) -> pd.DataFrame:
+    filtered_df = dataf.copy()
+    conditions = []
+
+    for val in selected_types:
+        if val == 'Dense single-vector bi-encoder (DSVBE)':
+            conditions.append((filtered_df['Type'] == 'DSVBE'))
+        elif val == 'Dense multi-vector bi-encoder (DMVBE)':
+            conditions.append((filtered_df['Type'] == 'DMVBE'))
+        elif val == 'Sparse single-vector bi-encoder (SSVBE)':
+            conditions.append((filtered_df['Type'] == 'SSVBE'))
+        elif val == 'Cross-encoder (CE)':
+            conditions.append((filtered_df['Type'] == 'CE'))
+        elif val == 'LLM':
+            conditions.append((filtered_df['Type'] == 'LLM'))
+    
+    for val in selected_sizes:
+        if val == 'Small (< 100M)':
+            conditions.append((filtered_df['#Params (M)'] < 100))
+        elif val == 'Base (100M-300M)':
+            conditions.append((filtered_df['#Params (M)'] >= 100) & (filtered_df['#Params (M)'] <= 300))
+        elif val == 'Large (300M-500M)':
+            conditions.append((filtered_df['#Params (M)'] >= 300) & (filtered_df['#Params (M)'] <= 500))
+        elif val == 'Extra-large (500M+)':
+            conditions.append((filtered_df['#Params (M)'] > 500))
+    
+    if conditions:
+        filtered_df = filtered_df[pd.concat(conditions, axis=1).any(axis=1)]
+    
+    if query:
+        filtered_df = filtered_df[filtered_df['Model'].str.contains(query, case=False)]
+    
+    return filtered_df
+
+
+with gr.Blocks() as demo:
+    gr.HTML("""
+        <div style="display: flex; flex-direction: column; align-items: center;">
+            <div style="align-self: flex-start;">
+                <a href="mailto:[email protected]" target="_blank" style="color: blue; text-decoration: none;">Contact/Submissions</a>
+            </div>
+            <h1 style="margin: 0;">🥇 DécouvrIR\n</h1>A Benchmark for Evaluating the Robustness of Information Retrieval Models in French</h1>
+        </div>
+    """)
+
+    # Create the Pandas dataframes (one per dataset)
+    all_df = load_all_results()
+    mmarco_df = filter_dataf_by_dataset(all_df, dataset_name="mMARCO-fr", sort_by="Recall@500")
+    bsard_df = filter_dataf_by_dataset(all_df, dataset_name="BSARD", sort_by="Recall@500")
+    
+    # Search and filter widgets
+    with gr.Column():
+        with gr.Row():
+            search_bar = gr.Textbox(placeholder=" 🔍 Search for a model...", show_label=False, elem_id="search-bar")
+        
+        with gr.Row():
+            filter_type = gr.CheckboxGroup(
+                label="Model type",
+                choices=[
+                    'Dense single-vector bi-encoder (DSVBE)', 
+                    'Dense multi-vector bi-encoder (DMVBE)', 
+                    'Sparse single-vector bi-encoder (SSVBE)', 
+                    'Cross-encoder (CE)', 
+                    'LLM',
+                ],
+                value=[],
+                interactive=True,
+                elem_id="filter-type",
+            )
+
+        with gr.Row():
+            filter_size = gr.CheckboxGroup(
+                label="Model size",
+                choices=['Small (< 100M)', 'Base (100M-300M)', 'Large (300M-500M)', 'Extra-large (500M+)'],
+                value=[],
+                interactive=True,
+                elem_id="filter-size",
+            )
+
+    # Leaderboard tables
+    with gr.Tabs():
+        with gr.TabItem("🌐 mMARCO-fr"):
+            gr.HTML("""
+                <p>The <a href="https://huggingface.co/datasets/unicamp-dl/mmarco" target="_blank" style="color: blue; text-decoration: none;">mMARCO</a> dataset is a machine-translated version of 
+                the widely popular MS MARCO dataset across 13 languages (including French) for studying <strong> domain-general</strong> passage retrieval.</p>
+                <p>The evaluation is performed on <strong>6,980 dev questions</strong> labeled with relevant passages to be retrieved from a corpus of <strong>8,841,823 candidates</strong>.</p>
+            """)
+            mmarco_table = gr.Dataframe(
+                value=mmarco_df,
+                datatype=[COLUMNS[col] for col in mmarco_df.columns],
+                interactive=False,
+                elem_classes="text-sm",
+            )
+
+        with gr.TabItem("⚖️ BSARD"):
+            gr.HTML("""
+                <p>The <a href="https://huggingface.co/datasets/maastrichtlawtech/bsard" target="_blank" style="color: blue; text-decoration: none;">Belgian Statutory Article Retrieval Dataset (BSARD)</a> is a 
+                French native dataset for studying <strong>legal</strong> document retrieval.</p>
+                <p>The evaluation is performed on <strong>222 test questions</strong> labeled by experienced jurists with relevant Belgian law articles to be retrieved from a corpus of <strong>22,633 candidates</strong>.</p>
+                <i>[Coming soon...]</i>
+            """)
+            # bsard_table = gr.Dataframe(
+            #     value=bsard_df,
+            #     datatype=[COLUMNS[col] for col in bsard_df.columns],
+            #     interactive=False,
+            #     elem_classes="text-sm",
+            # )
+
+    # Update tables on search.
+    search_bar.change(
+        fn=lambda x: update_table(dataf=mmarco_df, query=x, selected_types=filter_type.value, selected_sizes=filter_size.value),
+        inputs=[search_bar],
+        outputs=mmarco_table,
+    )
+    # search_bar.change(
+    #     fn=lambda x: update_table(dataf=bsard_df, query=x, selected_types=filter_type.value, selected_sizes=filter_size.value),
+    #     inputs=[search_bar],
+    #     outputs=bsard_table,
+    # )
+
+    # Update tables on model type filter.
+    filter_type.change(
+        fn=lambda selected_types: update_table(mmarco_df, search_bar.value, selected_types, filter_size.value),
+        inputs=[filter_type],
+        outputs=mmarco_table,
+    )
+    # filter_type.change(
+    #     fn=lambda selected_types: update_table(bsard_df, search_bar.value, selected_types, filter_size.value),
+    #     inputs=[filter_type],
+    #     outputs=bsard_table,
+    # )
+
+    # Update tables on model size filter.
+    filter_size.change(
+        fn=lambda selected_sizes: update_table(mmarco_df, search_bar.value, filter_type.value, selected_sizes),
+        inputs=[filter_size],
+        outputs=mmarco_table,
+    )
+    # filter_size.change(
+    #     fn=lambda selected_sizes: update_table(bsard_df, search_bar.value, filter_type.value, selected_sizes),
+    #     inputs=[filter_size],
+    #     outputs=bsard_table,
+    # )
+    
+    # Citation
+    with gr.Column():
+        with gr.Row():
+            gr.HTML("""
+                <h2>Citation</h2>
+                <p>For attribution in academic contexts, please cite this benchmark and any of the models released by <a href="https://huggingface.co/antoinelouis" target="_blank" style="color: blue; text-decoration: none;">@antoinelouis</a> as follows:</p>
+            """)
+        with gr.Row():
+            citation_block = (
+                "@online{louis2024decouvrir,\n"
+                "\tauthor    = 'Antoine Louis',\n"
+                "\ttitle     = 'DécouvrIR: A Benchmark for Evaluating the Robustness of Information Retrieval Models in French',\n"
+                "\tpublisher = 'Hugging Face',\n"
+                "\tmonth     = 'mar',\n"
+                "\tyear      = '2024',\n"
+                "\turl       = 'https://huggingface.co/spaces/antoinelouis/decouvrir',\n"
+                "}\n"
+            )
+            gr.Code(citation_block, language=None, show_label=False)
+
+demo.launch()