app: add initial version

app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+import pandas as pd
+
+title = """
+# hmLeaderboard
+
+![hmLeaderboard](logo.png)
+"""
+
+description = """
+## Space for tracking and ranking models on Historic NER Datasets.
+
+At the moment the following models are supported:
+
+* hmBERT: [Historical Multilingual Language Models for Named Entity Recognition](https://huggingface.co/hmbert).
+* hmTEAMS: [Historic Multilingual TEAMS Models](https://huggingface.co/hmteams).
+"""
+footer = "Made from Bavarian Oberland with ❤️ and 🥨."
+
+model_selection_file_names = {
+    "Best Configuration": "best_model_configurations.csv",
+    "Best Model": "best_models.csv"
+}
+
+df_init = pd.read_csv(model_selection_file_names["Best Configuration"])
+dataset_names = df_init.columns.values[1:].tolist()
+languages = list(set([dataset_name.split(" ")[0] for dataset_name in dataset_names]))
+
+
+def perform_evaluation_for_datasets(model_selection, selected_datasets):
+    df = pd.read_csv(model_selection_file_names.get(model_selection))
+
+    selected_indices = []
+
+    for selected_dataset in selected_datasets:
+        selected_indices.append(dataset_names.index(selected_dataset) + 1)
+
+    mean_column = df.iloc[:, selected_indices].mean(axis=1).round(2)
+
+    # Include column with column name
+    result_df = df.iloc[:, [0] + selected_indices]
+    result_df["Average"] = mean_column
+
+    return result_df
+
+def perform_evaluation_for_languages(model_selection, selected_languages):
+    df = pd.read_csv(model_selection_file_names.get(model_selection))
+
+    selected_indices = []
+
+    for selected_language in selected_languages:
+        selected_language = selected_language.lower()
+        found_indices = [i for i, column_name in enumerate(df.columns) if selected_language in column_name.lower()]
+
+        for found_index in found_indices:
+            selected_indices.append(found_index)
+
+    mean_column = df.iloc[:, selected_indices].mean(axis=1).round(2)
+
+    # Include column with column name
+    result_df = df.iloc[:, [0] + selected_indices]
+    result_df["Average"] = mean_column
+
+    return result_df
+
+with gr.Blocks() as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+
+    with gr.Tab("Overview"):
+        gr.Markdown("### Best Configuration\nThe best hyper-parameter configuration for each model is used and average F1-score over runs with different seeds is reported here:")
+
+        df_result = perform_evaluation_for_datasets("Best Configuration", dataset_names)
+
+        gr.Dataframe(value=df_result)
+
+        gr.Markdown("### Best Model\nThe best hyper-parameter configuration for each model is used and the model with highest F1-score is used and its performance is reported here:")
+
+        df_result = perform_evaluation_for_datasets("Best Model", dataset_names)
+
+        gr.Dataframe(value=df_result)
+
+    with gr.Tab("Filtering"):
+
+        gr.Markdown("### Filtering\nSwiss-knife filtering for single datasets and languages is possible.")
+
+        model_selection = gr.Radio(choices=["Best Configuration", "Best Model"],
+                                   label="Model Selection",
+                                   info="Defines if best configuration or best model should be used for evaluation. When 'Best Configuration' is used, the best hyper-parameter configuration is used and then averaged F1-score over all runs is calculated. When 'Best Model' is chosen, the best hyper-parameter configuration and model with highest F1-score on development dataset is used (best model).",
+                                   value="Best Configuration")
+
+        with gr.Tab("Dataset Selection"):
+            datasets_selection = gr.CheckboxGroup(
+                dataset_names, label="Datasets", info="Select datasets for evaluation"
+            )
+            output_df = gr.Dataframe()
+
+            evaluation_button = gr.Button("Evaluate")
+            evaluation_button.click(fn=perform_evaluation_for_datasets, inputs=[model_selection, datasets_selection], outputs=output_df)
+
+
+        with gr.Tab("Language Selection"):
+            language_selection = gr.CheckboxGroup(
+                languages, label="Languages", info="Select languages for evaluation"
+            )
+            output_df = gr.Dataframe()
+
+            evaluation_button = gr.Button("Evaluate")
+            evaluation_button.click(fn=perform_evaluation_for_languages, inputs=[model_selection, language_selection], outputs=output_df)
+
+
+
+    gr.Markdown(footer)
+
+demo.launch()