Create utils.py

utils.py

@@ -0,0 +1,237 @@
+import json
+import os
+from datetime import datetime
+
+import gradio as gr
+import pandas as pd
+
+from envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+
+
+
+custom_css = """
+@import url('https://fonts.googleapis.com/css2?family=Vazirmatn&display=swap');
+body, .gradio-container, .gr-button, .gr-input, .gr-slider, .gr-dropdown, .gr-markdown {
+    font-family: 'Vazirmatn', sans-serif !important;
+}
+.markdown-text {
+    font-size: 16px !important;
+}
+#models-to-add-text {
+    font-size: 18px !important;
+}
+#citation-button span {
+    font-size: 16px !important;
+}
+#citation-button textarea {
+    font-size: 16px !important;
+}
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+#leaderboard-table {
+    margin-top: 15px;
+    text-align: center;
+}
+#leaderboard-table,
+#leaderboard-table th,
+#leaderboard-table td {
+    text-align: center;
+    vertical-align: middle;
+    border-collapse: collapse;
+}
+#leaderboard-table td:first-child,
+#leaderboard-table th:first-child {
+    text-align: left;
+    max-width: 600px;
+}
+table > thead {
+    white-space: normal;
+}
+table > thead th,
+table > tbody td {
+    text-align: center;
+    vertical-align: middle;
+}
+table > tbody td:first-child {
+    text-align: left;
+    max-width: 600px;
+}
+#leaderboard-table-lite {
+    margin-top: 15px;
+}
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+#search-bar {
+    padding: 0px;
+}
+.tab-buttons button {
+    font-size: 20px;
+}
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+#scale-logo .download {
+    display: none;
+}
+#filter_type {
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span {
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap {
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner {  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input {
+    width: 1px;
+}
+#filter-columns-type {
+    border: 0;
+    padding: 0.5;
+}
+#filter-columns-size {
+    border: 0;
+    padding: 0.5;
+}
+#box-filter > .form {
+    border: 0;
+}
+"""
+
+
+ABOUT_TEXT = f"""
+# Persian Text Embedding Benchmark (v1.0.0)
+"""
+
+
+SUBMIT_TEXT = """## Submitting a Model for Evaluation
+
+> To submit your open-source model for evaluation, follow these steps:
+> 
+> 1. **Ensure your model is on Hugging Face**: Your model must be publicly available on [Hugging Face](https://huggingface.co/).
+> 
+> 2. **Submit Request**: Send a request with your model's Hugging Face identifier.
+> 
+> 3. **Manual Queue**: Please note that the evaluation process is currently handled manually. Submissions will be queued and processed as soon as possible.
+> 
+> 4. **Results**: Once the evaluation is complete, your model’s results will be updated on the leaderboard.
+> 
+> We appreciate your patience and contributions to the Persian LM ecosystem!
+"""
+
+
+PART_LOGO = """
+<img src="https://avatars.githubusercontent.com/u/39557177?v=4" style="width:30%;display:block;margin-left:auto;margin-right:auto"> 
+<h1 style="font-size: 28px; margin-bottom: 2px;">Part DP AI</h1>
+"""
+
+
+def load_jsonl(input_file):
+    data = []
+    with open(input_file, 'r') as f:
+        for line in f:
+            data.append(json.loads(line))
+    return data
+
+
+def jsonl_to_dataframe(input_file):
+    data = load_jsonl(input_file)
+    return pd.DataFrame(data)
+
+
+def sort_dataframe_by_column(df, column_name):
+    if column_name not in df.columns:
+        raise ValueError(f"Column '{column_name}' does not exist in the DataFrame.")
+    return df.sort_values(by=column_name, ascending=False).reset_index(drop=True)
+
+
+def add_average_column_to_df(df,columns_to_average, index=3, average_column_name="Average Accuracy"):
+    average_column = df[columns_to_average].mean(axis=1)
+    df.insert(index, average_column_name, average_column)
+    return df
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def center_align_markdown(text):
+    return f'<div align="center">{text}</div>'
+
+
+def apply_markdown_format_for_columns(df, model_column_name):
+    columns = list(df.columns)
+    df[model_column_name] = df[model_column_name].apply(make_clickable_model)
+    # for column in columns:
+    #     if column != model_column_name:
+    #         df[column] = df[column].apply(center_align_markdown)
+    return df
+
+
+def submit(model_name, model_id, contact_email):
+    if model_name == "" or model_id == "" or contact_email == "":
+        gr.Info("Please fill all the fields")
+        return
+
+
+    try:
+        user_name = ""
+        if "/" in model_id:
+            user_name = model_id.split("/")[0]
+            model_path = model_id.split("/")[1]
+
+        eval_entry = {
+            "model_name": model_name,
+            "model_id": model_id,
+            "contact_email": contact_email,
+        }
+
+        # Get the current timestamp to add to the filename
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(OUT_DIR, exist_ok=True)
+
+        # Add the timestamp to the filename
+        out_path = f"{OUT_DIR}/{user_name}_{model_path}_{timestamp}.json"
+
+        with open(out_path, "w") as f:
+            f.write(json.dumps(eval_entry))
+
+        print("Uploading eval file")
+        API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split("eval-queue/")[1],
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model_name} to eval queue",
+        )
+
+        gr.Info("Successfully submitted", duration=10)
+        # Remove the local file
+        os.remove(out_path)
+    except Exception as e:
+        gr.Error(f"Error submitting the model: {e}")