utils.py

import pandas as pd
import gradio as gr
import csv
import json
import os
import shutil
from huggingface_hub import Repository

HF_TOKEN = os.environ.get("HF_TOKEN")

MODEL_INFO = [
    "Model (CoT)",
    "Avg",
    "TheoremQA",
    "MATH",
    "GSM",
    "GPQA",
    "MMLU-STEM"
    ]

DATA_TITILE_TYPE = ['markdown', 'number', 'number', 'number', 'number', 'number', 'number']

SUBMISSION_NAME = "science_leaderboard_submission"
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/wenhu/", SUBMISSION_NAME)
CSV_DIR = "./science_leaderboard_submission/results.csv"

COLUMN_NAMES = MODEL_INFO

LEADERBORAD_INTRODUCTION = """# Science Leaderboard
    
    **"Which large language model is the BEST on scinece and engineering?"**<br>
    🏆 Welcome to the **Science** leaderboard! The leaderboard covers the most popular evaluation for different science subjects including math, phyiscs, biology, chemistry, computer science, finance.
    <div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
    </div>
    The evaluation set from the following datasets are being included in the leaderboard.
    <ul>
        <li> MATH (4-shot): this contains the test set of 5000 questions from American Math contest covering different fields like algebra, calculus, statistics, geometry, linear algebra, number theory.
        <li> GSM8K (4-shot): this contains the test set of 1320 questions from grade school math word problems. This dataset is mainly covering algebra problems.
        <li> TheoremQA (5-shot): this contains the test set of 800 questions collected from college-level exams. This covers math, physics, engineering and finance.
        <li> GPQA (5-shot): this contains the test of 198 questions from college-level dataset GPQA-diamond. This covers many fields like chemistry, genetics, biology, etc.
        <li> MMLU-STEM (5-shot): this contains the test of 3.3K questions from MMLU dataset. This covers many fields like math, chemistry, genetics, biology, computer science, anatomy, astronomy, etc.
    </ul>

    **"How to evaluate your model and submit your results?"**<br>
    Please refer to the guideline in <a href="https://github.com/TIGER-AI-Lab/MAmmoTH/blob/main/math_eval/README.md">Github</a> to evaluate your own model.

    <a href='https://hits.seeyoufarm.com'><img src='https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fhuggingface.co%2Fspaces%2FTIGER-Lab%2FTheoremQA-Leaderboard&count_bg=%23C7C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false'></a>
    """

TABLE_INTRODUCTION = """
    """

LEADERBORAD_INFO = """
We list the information of the used datasets as follows:<br>

MATH: Measuring Mathematical Problem Solving With the MATH Dataset<br>
<a href='https://arxiv.org/pdf/2103.03874.pdf'>Paper</a><br>
<a href='https://github.com/hendrycks/math'>Code</a><br>

GSM8K: Training Verifiers to Solve Math Word Problems<br>
<a href='https://arxiv.org/pdf/2110.14168.pdf'>Paper</a><br>
<a href='https://github.com/openai/grade-school-math'>Code</a><br>

TheoremQA: A Theorem-driven Question Answering dataset<br>
<a href='https://arxiv.org/pdf/2305.12524.pdf'>Paper</a><br>
<a href='https://github.com/TIGER-AI-Lab/TheoremQA'>Code</a><br>

GPQA: A Graduate-Level Google-Proof Q&A Benchmark<br>
<a href='https://arxiv.org/pdf/2311.12022.pdf'>Paper</a><br>
<a href='https://github.com/idavidrein/gpqa'>Code</a>

MMLU: Measuring Massive Multitask Language Understanding<br>
<a href='https://arxiv.org/pdf/2009.03300.pdf'>Paper</a><br>
<a href='https://github.com/hendrycks/test'>Code</a>
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""@inproceedings{hendrycks2021measuring,
  title={Measuring Mathematical Problem Solving With the MATH Dataset},
  author={Hendrycks, Dan and Burns, Collin and Kadavath, Saurav and Arora, Akul and Basart, Steven and Tang, Eric and Song, Dawn and Steinhardt, Jacob},
  booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)},
  year={2021}
}
@article{cobbe2021training,
  title={Training verifiers to solve math word problems},
  author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and others},
  journal={arXiv preprint arXiv:2110.14168},
  year={2021}
}
@inproceedings{chen2023theoremqa,
  title={Theoremqa: A theorem-driven question answering dataset},
  author={Chen, Wenhu and Yin, Ming and Ku, Max and Lu, Pan and Wan, Yixin and Ma, Xueguang and Xu, Jianyu and Wang, Xinyi and Xia, Tony},
  booktitle={The 2023 Conference on Empirical Methods in Natural Language Processing},
  year={2023}
}
@article{rein2023gpqa,
  title={Gpqa: A graduate-level google-proof q\&a benchmark},
  author={Rein, David and Hou, Betty Li and Stickland, Asa Cooper and Petty, Jackson and Pang, Richard Yuanzhe and Dirani, Julien and Michael, Julian and Bowman, Samuel R},
  journal={arXiv preprint arXiv:2311.12022},
  year={2023}
}
@inproceedings{hendrycks2020measuring,
  title={Measuring Massive Multitask Language Understanding},
  author={Hendrycks, Dan and Burns, Collin and Basart, Steven and Zou, Andy and Mazeika, Mantas and Song, Dawn and Steinhardt, Jacob},
  booktitle={International Conference on Learning Representations},
  year={2020}
}"""

SUBMIT_INTRODUCTION = """# Submit on Science Leaderboard Introduction

## ⚠ Please note that you need to submit the json file with following format:

```json
{
    "Model": "[NAME]",
    "Repo": "https://huggingface.co/[MODEL_NAME]"
    "TheoremQA": 50,
    "MATH": 50,
    "GSM": 50,
    "GPQA": 50,
    "MMLU-STEM": 50
}
```
After submitting, you can click the "Refresh" button to see the updated leaderboard(it may takes few seconds).

"""
def get_df():
    repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN)
    repo.git_pull()
    df = pd.read_csv(CSV_DIR)
    df['Avg'] = df[['TheoremQA', 'MATH', 'GSM', 'GPQA', 'MMLU-STEM']].mean(axis=1).round(1)
    df = df.sort_values(by=['Avg'], ascending=False)
    return df[COLUMN_NAMES]

def add_new_eval(
    input_file,
):
    if input_file is None:
        return "Error! Empty file!"

    upload_data=json.loads(input_file)
    data_row = [f'[{upload_data["Model"]}]({upload_data["Repo"]})', upload_data['TheoremQA'], upload_data['MATH'], upload_data['GSM'], upload_data['GPQA'], upload_data['MMLU-STEM']]

    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
    submission_repo.git_pull()

    already_submitted = []
    with open(CSV_DIR, mode='r') as file:
        reader = csv.reader(file, delimiter=',')
        for row in reader:
            already_submitted.append(row[0])

    if data_row[0] not in already_submitted:
        with open(CSV_DIR, mode='a', newline='') as file:
            writer = csv.writer(file)
            writer.writerow(data_row)
        
        submission_repo.push_to_hub()
        print('Submission Successful')
    else:
        print('The entry already exists')


def refresh_data():
    return get_df()