Translation both from and to

evals/backend.py

@@ -22,7 +22,7 @@ def mean(lst):
     return sum(lst) / len(lst) if lst else None
 
 
-task_metrics = ["translation_bleu", "classification_accuracy"]
+task_metrics = ["translation_from_bleu", "translation_to_bleu", "classification_accuracy"]
 
 
 def make_model_table(df, models):

evals/models.py

@@ -116,7 +116,7 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
             raise ValueError(f"Model {model} not supported")
 
 
-models = pd.DataFrame(models, columns=["id"])
+models = pd.DataFrame(models, columns=["id"]).iloc[:3]
 
 
 @cache
@@ -144,7 +144,6 @@ def get_hf_metadata(row):
     if not row:
         return empty
     id = row["hf_slug"] or row["slug"].split(":")[0]
-    print(id)
     if not id:
         return empty
     try:

evals/tasks.py

@@ -1,30 +1,37 @@
 import random
+from functools import partial
 
 import evaluate
 import pandas as pd
+import sentencepiece as spm
+from datasets_.flores import flores_sentences
 from joblib.memory import Memory
 from languages import languages, script_name
-from datasets_.flores import flores_sentences
 from models import complete, transcribe
-import sentencepiece as spm
 
 cache = Memory(location=".cache", verbose=0).cache
 bleu = evaluate.load("bleu")
 chrf = evaluate.load("chrf")
 wer = evaluate.load("wer")
-tokenizer = spm.SentencePieceProcessor(model_file="data/spbleu/flores200_sacrebleu_tokenizer_spm.model")
+tokenizer = spm.SentencePieceProcessor(
+    model_file="data/spbleu/flores200_sacrebleu_tokenizer_spm.model"
+)
 
 # sample languages to translate to
 target_languages = languages[languages["in_benchmark"]].sample(
     frac=1, weights="speakers", replace=True, random_state=42
 )
 
+
 @cache
-async def translate_and_evaluate(model, original_language_bcp_47, sentence_nr):
-    original_language = languages[languages["bcp_47"] == original_language_bcp_47].iloc[
-        0
-    ]
+async def translate_and_evaluate(model, bcp_47, sentence_nr, mode="from"):
+    original_language = languages[languages["bcp_47"] == bcp_47].iloc[0]
     target_language = target_languages.iloc[sentence_nr]
+    match mode:
+        case "from":
+            pass
+        case "to":
+            original_language, target_language = target_language, original_language
     original_sentence = flores_sentences(original_language)[sentence_nr].strip()
     target_sentence = flores_sentences(target_language)[sentence_nr].strip()
     script = script_name(target_language.flores_path.split("_")[1])
@@ -52,14 +59,15 @@ async def translate_and_evaluate(model, original_language_bcp_47, sentence_nr):
     return [
         {
             "model": model,
-            "bcp_47": original_language["bcp_47"],
-            "task": "translation",
+            "bcp_47": bcp_47,
+            "task": f"translation_{mode}",
             "metric": metric,
             "score": score,
             "sentence_nr": sentence_nr,
         }
-        for metric, score in zip(
-            ["bleu", "chrf"], [bleu_score["bleu"], chrf_score["score"] / 100]
+        for metric, score in (
+            ("bleu", bleu_score["bleu"]),
+            ("chrf", chrf_score["score"] / 100),
         )
     ]
 
@@ -68,8 +76,8 @@ metadata = pd.read_csv("data/floresp-v2.0-rc.3/metadata_dev.tsv", sep="\t")
 
 
 @cache
-async def classify_and_evaluate(model, language_bcp_47, nr):
-    language = languages[languages["bcp_47"] == language_bcp_47].iloc[0]
+async def classify_and_evaluate(model, bcp_47, nr):
+    language = languages[languages["bcp_47"] == bcp_47].iloc[0]
     sentences = pd.DataFrame(flores_sentences(language), columns=["text"])
     sentences = pd.concat([metadata, sentences], axis=1)
     sentences = sentences.dropna(subset=["topic"])
@@ -119,7 +127,7 @@ async def classify_and_evaluate(model, language_bcp_47, nr):
     return [
         {
             "model": model,
-            "bcp_47": language["bcp_47"],
+            "bcp_47": bcp_47,
             "task": "classification",
             "metric": "accuracy",
             "score": int(pred == true),
@@ -177,6 +185,7 @@ async def mlm_and_evaluate(model, language_bcp_47, nr):
         }
     ]
 
+
 @cache
 async def transcribe_and_evaluate(model, language_bcp_47, nr):
     language = languages[languages["bcp_47"] == language_bcp_47].iloc[0]
@@ -210,8 +219,9 @@ async def transcribe_and_evaluate(model, language_bcp_47, nr):
 
 
 tasks = [
-    translate_and_evaluate,
+    partial(translate_and_evaluate, mode="from"),
+    partial(translate_and_evaluate, mode="to"),
     classify_and_evaluate,
     # mlm_and_evaluate,
     # transcribe_and_evaluate,
-]
+]

frontend/src/components/LanguageTable.js

@@ -174,10 +174,22 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
       <Column
-        field='translation_bleu'
-        header='Translation'
+        field='translation_from_bleu'
+        header="Translation (from)"
+        headerTooltip='Translation performance from a language to all other languages (spBLEU score)'
         sortable
-        body={scoreBodyTemplate('translation_bleu', {
+        body={scoreBodyTemplate('translation_from_bleu', {
+          minScore: 0,
+          maxScore: 0.5
+        })}
+        style={{ minWidth: '5rem', maxWidth: '10rem' }}
+      />
+      <Column
+        field='translation_to_bleu'
+        header="Translation (to)"
+        headerTooltip='Translation performance from all other languages to a language (spBLEU score)'
+        sortable
+        body={scoreBodyTemplate('translation_to_bleu', {
           minScore: 0,
           maxScore: 0.5
         })}

frontend/src/components/ModelTable.js

@@ -224,12 +224,24 @@ const ModelTable = ({ data }) => {
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
       <Column
-        field='translation_bleu'
-        header='Translation'
+        field='translation_from_bleu'
+        header="Translation (from)"
+        headerTooltip='Translation performance from a language to all other languages (spBLEU score)'
         sortable
-        body={scoreBodyTemplate('translation_bleu', {
+        body={scoreBodyTemplate('translation_from_bleu', {
           minScore: 0,
-          maxScore: 0.3
+          maxScore: 0.5
+        })}
+        style={{ minWidth: '5rem', maxWidth: '10rem' }}
+      />
+      <Column
+        field='translation_to_bleu'
+        header="Translation (to)"
+        headerTooltip='Translation performance from all other languages to a language (spBLEU score)'
+        sortable
+        body={scoreBodyTemplate('translation_to_bleu', {
+          minScore: 0,
+          maxScore: 0.5
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />