Refactor code for adding generic tasks

src/display/utils.py

@@ -12,14 +12,16 @@ class Task:
     benchmark: str
     metric: str
     col_name: str
+    baseline: float = 0.0
+    human_baseline: float = 0.0
 
 class Tasks(Enum):
-    arc = Task("arc:challenge", "acc_norm", "ARC")
-    hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
-    mmlu = Task("hendrycksTest", "acc", "MMLU")
-    truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
-    winogrande = Task("winogrande", "acc", "Winogrande")
-    gsm8k = Task("gsm8k", "acc", "GSM8K")
+    arc = Task("arc:challenge", "acc_norm", "ARC", 25.0, 80.0)
+    hellaswag = Task("hellaswag", "acc_norm", "HellaSwag", 25.0, 95.0)
+    mmlu = Task("hendrycksTest", "acc", "MMLU", 25.0, 89.8)
+    truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA", 25.0, 94.0)
+    winogrande = Task("winogrande", "acc", "Winogrande", 50.0, 94.0)
+    gsm8k = Task("gsm8k", "acc", "GSM8K", 0.21, 100)
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
@@ -75,26 +77,33 @@ baseline_row = {
     AutoEvalColumn.revision.name: "N/A",
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.merged.name: False,
-    AutoEvalColumn.average.name: 31.0,
-    AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name: 25.0,
-    AutoEvalColumn.mmlu.name: 25.0,
-    AutoEvalColumn.truthfulqa.name: 25.0,
-    AutoEvalColumn.winogrande.name: 50.0,
-    AutoEvalColumn.gsm8k.name: 0.21,
+    #AutoEvalColumn.average.name: 31.0,
+    #AutoEvalColumn.arc.name: 25.0,
+    #AutoEvalColumn.hellaswag.name: 25.0,
+    #AutoEvalColumn.mmlu.name: 25.0,
+    #AutoEvalColumn.truthfulqa.name: 25.0,
+    #AutoEvalColumn.winogrande.name: 50.0,
+    #AutoEvalColumn.gsm8k.name: 0.21,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
     AutoEvalColumn.model_type_symbol.name: None,
     AutoEvalColumn.architecture.name: None,
     AutoEvalColumn.weight_type.name: None,
-    AutoEvalColumn.params.name: None,
-    AutoEvalColumn.likes.name: None,
-    AutoEvalColumn.license.name: None,
-    AutoEvalColumn.still_on_hub.name: None,
-    AutoEvalColumn.moe.name: None
+    AutoEvalColumn.params.name: 0,
+    AutoEvalColumn.likes.name: 0,
+    AutoEvalColumn.license.name: "",
+    AutoEvalColumn.still_on_hub.name: False,
+    AutoEvalColumn.moe.name: False
 }
 
+baseline_list = []
+for task in Tasks:
+    baseline_row[task.name] = task.value.baseline
+    if task.value.baseline is not None:
+        baseline_list.append(task.value.baseline)
+baseline_row[AutoEvalColumn.average.name] = round(sum(baseline_list) / len(baseline_list), 2)
+
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
 # ARC human baseline is 0.80 (source: https://lab42.global/arc/)
 # HellaSwag human baseline is 0.95 (source: https://deepgram.com/learn/hellaswag-llm-benchmark-guide)
@@ -107,19 +116,34 @@ human_baseline_row = {
     AutoEvalColumn.model.name: "<p>Human performance</p>",
     AutoEvalColumn.revision.name: "N/A",
     AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 92.75,
+    #AutoEvalColumn.average.name: 92.75,
     AutoEvalColumn.merged.name: False,
-    AutoEvalColumn.arc.name: 80.0,
-    AutoEvalColumn.hellaswag.name: 95.0,
-    AutoEvalColumn.mmlu.name: 89.8,
-    AutoEvalColumn.truthfulqa.name: 94.0,
-    AutoEvalColumn.winogrande.name: 94.0,
-    AutoEvalColumn.gsm8k.name: 100,
+    #AutoEvalColumn.arc.name: 80.0,
+    #AutoEvalColumn.hellaswag.name: 95.0,
+    #AutoEvalColumn.mmlu.name: 89.8,
+    #AutoEvalColumn.truthfulqa.name: 94.0,
+    #AutoEvalColumn.winogrande.name: 94.0,
+    #AutoEvalColumn.gsm8k.name: 100,
     AutoEvalColumn.dummy.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
+    AutoEvalColumn.model_type_symbol.name: None,
+    AutoEvalColumn.architecture.name: None,
+    AutoEvalColumn.weight_type.name: None,
+    AutoEvalColumn.params.name: 0,
+    AutoEvalColumn.likes.name: 0,
+    AutoEvalColumn.license.name: "",
+    AutoEvalColumn.still_on_hub.name: False,
+    AutoEvalColumn.moe.name: False
 }
 
+baseline_list = []
+for task in Tasks:
+    human_baseline_row[task.name] = task.value.human_baseline
+    if task.value.human_baseline is not None:
+        baseline_list.append(task.value.human_baseline)
+human_baseline_row[AutoEvalColumn.average.name] = round(sum(baseline_list) / len(baseline_list), 2)
+
 @dataclass
 class ModelDetails:
     name: str

src/leaderboard/read_evals.py

@@ -66,6 +66,7 @@ class EvalResult:
         results = {}
         for task in Tasks:
             task = task.value
+            """
             # We skip old mmlu entries
             wrong_mmlu_version = False
             if task.benchmark == "hendrycksTest":
@@ -81,11 +82,12 @@ class EvalResult:
                 if math.isnan(float(data["results"]["harness|truthfulqa:mc|0"][task.metric])):
                     results[task.benchmark] = 0.0
                     continue
-
+            """
             # We average all scores of a given metric (mostly for mmlu)
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
+            
 
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc