Run on 100 languages, adjust display

evals/backend.py

@@ -20,6 +20,7 @@ models = pd.DataFrame(results["models"])
 def mean(lst):
     return sum(lst) / len(lst) if lst else None
 
+task_metrics = ["translation_bleu", "classification_accuracy"]
 
 def make_model_table(df, models):
     df = (
@@ -29,7 +30,6 @@ def make_model_table(df, models):
     )
     df["task_metric"] = df["task"] + "_" + df["metric"]
     df = df.drop(columns=["task", "metric"])
-    task_metrics = df["task_metric"].unique()
     df = df.pivot(index="model", columns="task_metric", values="score").fillna(0)
     df["average"] = df[task_metrics].mean(axis=1)
     df = df.sort_values(by="average", ascending=False).reset_index()
@@ -59,7 +59,6 @@ def make_language_table(df, languages):
     )
     df["task_metric"] = df["task"] + "_" + df["metric"]
     df = df.drop(columns=["task", "metric"])
-    task_metrics = df["task_metric"].unique()
     df = (
         df.pivot(index="bcp_47", columns="task_metric", values="score")
         .fillna(0)

evals/main.py

@@ -6,16 +6,12 @@ import pandas as pd
 from tqdm.asyncio import tqdm_asyncio
 
 from languages import languages
-from models import model_fast, models
+from models import models
 from tasks import tasks
 
 # ===== config =====
 
-n_sentences = 30
-langs_eval = languages.iloc[:30]
-langs_eval_detailed = languages.iloc[:2]
-transcription_langs_eval = languages.iloc[:10]
-transcription_langs_eval_detailed = languages.iloc[:5]
+n_sentences = 10
 
 # ===== run evaluation and aggregate results =====
 
@@ -23,16 +19,12 @@ transcription_langs_eval_detailed = languages.iloc[:5]
 async def evaluate():
     print("running evaluations")
     results = [
-        task(model, original_language.bcp_47, i)
+        task(model, lang.bcp_47, i)
         for task in tasks
         for i in range(n_sentences)
-        for original_language in langs_eval.itertuples()
+        for lang in languages.iloc[:100].itertuples()
         for model in models["id"]
-        if original_language.in_benchmark
-        and (
-            model == model_fast
-            or original_language.bcp_47 in langs_eval_detailed.bcp_47.values
-        )
+        if lang.in_benchmark
     ]
     return await tqdm_asyncio.gather(*results, miniters=1)
 

evals/models.py

@@ -14,23 +14,24 @@ from requests import HTTPError
 models = [
     "openai/gpt-4o-mini",  # 0.6$/M tokens
     # "anthropic/claude-3.5-haiku", # 4$/M tokens -> too expensive for dev
+    "meta-llama/llama-4-maverick", # 0.6$/M tokens
     "meta-llama/llama-3.3-70b-instruct",  # 0.3$/M tokens
     "meta-llama/llama-3.1-70b-instruct",  # 0.3$/M tokens
     "meta-llama/llama-3-70b-instruct", # 0.4$/M tokens
-    "mistralai/mistral-small-24b-instruct-2501",  # 0.14$/M tokens
-    "mistralai/mistral-nemo",
+    "mistralai/mistral-small-3.1-24b-instruct",  # 0.3$/M tokens
+    # "mistralai/mistral-saba", # 0.6$/M tokens
+    # "mistralai/mistral-nemo", # 0.08$/M tokens
     "google/gemini-2.0-flash-001",  # 0.4$/M tokens
-    "google/gemini-2.0-flash-lite-001",  # 0.3$/M tokens
+    # "google/gemini-2.0-flash-lite-001",  # 0.3$/M tokens
     "google/gemma-3-27b-it",  # 0.2$/M tokens
     # "qwen/qwen-turbo", # 0.2$/M tokens; recognizes "inappropriate content"
-    "qwen/qwq-32b",
-    # "deepseek/deepseek-chat",  # 1.3$/M tokens
+    "qwen/qwq-32b", # 0.2$/M tokens
+    "deepseek/deepseek-chat-v3-0324",  # 1.1$/M tokens
     # "microsoft/phi-4",  # 0.07$/M tokens; only 16k tokens context
-    "microsoft/phi-4-multimodal-instruct",
+    "microsoft/phi-4-multimodal-instruct", # 0.1$/M tokens
     "amazon/nova-micro-v1",  # 0.09$/M tokens
     # "openGPT-X/Teuken-7B-instruct-research-v0.4",  # not on OpenRouter
 ]
-model_fast = "meta-llama/llama-3.3-70b-instruct"
 
 transcription_models = [
     "elevenlabs/scribe_v1",
@@ -38,7 +39,6 @@ transcription_models = [
     # "openai/whisper-small",
     # "facebook/seamless-m4t-v2-large",
 ]
-transcription_model_fast = "elevenlabs/scribe_v1"
 
 load_dotenv()
 client = AsyncOpenAI(

evals/tasks.py

@@ -212,6 +212,6 @@ async def transcribe_and_evaluate(model, language_bcp_47, nr):
 tasks = [
     translate_and_evaluate,
     classify_and_evaluate,
-    mlm_and_evaluate,
+    # mlm_and_evaluate,
     # transcribe_and_evaluate,
 ]

frontend/src/components/LanguageTable.js

@@ -174,12 +174,12 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
       <Column
-        field='translation_chrf'
+        field='translation_bleu'
         header='Translation'
         sortable
-        body={scoreBodyTemplate('translation_chrf', {
-          minScore: 0.3,
-          maxScore: 0.6
+        body={scoreBodyTemplate('translation_bleu', {
+          minScore: 0,
+          maxScore: 0.5
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
@@ -188,12 +188,12 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
         header='Classification'
         sortable
         body={scoreBodyTemplate('classification_accuracy', {
-          minScore: 0.3,
-          maxScore: 0.7
+          minScore: 0,
+          maxScore: 0.5
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
-      <Column
+      {/* <Column
         field='language_modeling_chrf'
         header='Language Modeling'
         sortable
@@ -202,7 +202,7 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
           maxScore: 1
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
-      />
+      /> */}
     </DataTable>
   )
 }

frontend/src/components/ModelTable.js

@@ -103,18 +103,29 @@ const ModelTable = ({ data }) => {
     return <div style={{ textAlign: 'center' }}>{sizeStr}</div>
   }
 
-  const capitalize = s => String(s).charAt(0).toUpperCase() + String(s).slice(1)
+  const capitalize = s =>
+    (String(s).charAt(0).toUpperCase() + String(s).slice(1))
+      .replace(/gpt/i, 'GPT')
+      .replace(/qwq/i, 'QwQ')
+      .replace(/deepseek/i, 'DeepSeek')
+      .replace(/openai/i, 'OpenAI')
 
   const providerBodyTemplate = rowData => {
-    const providerName = rowData.model.split('/')[0].split('-').map(capitalize).join(' ')
+    const providerName = rowData.model
+      .split('/')[0]
+      .split('-')
+      .map(capitalize)
+      .join(' ')
     return providerName
   }
 
   const modelBodyTemplate = rowData => {
-    const modelName = rowData.model.split('/')[1].split('-').map(capitalize).join(' ')
-    return (
-      <div style={{ fontWeight: 'bold', height: '100%' }}>{modelName}</div>
-    )
+    const modelName = rowData.model
+      .split('/')[1]
+      .split('-')
+      .map(capitalize)
+      .join(' ')
+    return <div style={{ fontWeight: 'bold', height: '100%' }}>{modelName}</div>
   }
 
   const typeBodyTemplate = rowData => {
@@ -148,7 +159,12 @@ const ModelTable = ({ data }) => {
       style={{ width: '800px', minHeight: '650px' }}
     >
       <Column field='rank' body={rankBodyTemplate} />
-      <Column field='provider' header='Provider' style={{ minWidth: '5rem' }} body={providerBodyTemplate} />
+      <Column
+        field='provider'
+        header='Provider'
+        style={{ minWidth: '7rem' }}
+        body={providerBodyTemplate}
+      />
       <Column
         field='model'
         header='Model'
@@ -178,16 +194,16 @@ const ModelTable = ({ data }) => {
         field='average'
         header='Average'
         sortable
-        body={scoreBodyTemplate('average', { minScore: 0.3, maxScore: 0.6 })}
+        body={scoreBodyTemplate('average', { minScore: 0, maxScore: 0.6 })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
       <Column
-        field='translation_chrf'
+        field='translation_bleu'
         header='Translation'
         sortable
-        body={scoreBodyTemplate('translation_chrf', {
-          minScore: 0.3,
-          maxScore: 0.7
+        body={scoreBodyTemplate('translation_bleu', {
+          minScore: 0,
+          maxScore: 0.3
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
@@ -196,12 +212,12 @@ const ModelTable = ({ data }) => {
         header='Classification'
         sortable
         body={scoreBodyTemplate('classification_accuracy', {
-          minScore: 0.3,
-          maxScore: 0.8
+          minScore: 0,
+          maxScore: 0.9
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
-      <Column
+      {/* <Column
         field='language_modeling_chrf'
         header='Language Modeling'
         sortable
@@ -210,7 +226,7 @@ const ModelTable = ({ data }) => {
           maxScore: 1
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
-      />
+      /> */}
     </DataTable>
   )
 }