Display all languages and translate from multiple languages

bibliography.bib

@@ -243,6 +243,12 @@
   file = {/Users/david/Zotero/storage/VU6IFENR/Siminyu et al. - 2021 - AI4D -- African Language Program.pdf;/Users/david/Zotero/storage/7TV2PS8J/2104.html}
 }
 
+@misc{Tatoeba,
+  title = {Tatoeba},
+  urldate = {2024-11-03},
+  file = {/Users/david/Zotero/storage/4NDTCGWG/sentences_by_language.html}
+}
+
 @misc{teamNoLanguageLeft2022,
   title = {No {{Language Left Behind}}: {{Scaling Human-Centered Machine Translation}}},
   shorttitle = {No {{Language Left Behind}}},

index.html

@@ -39,42 +39,43 @@
             const scoreName = "BLEU Score"
             const chartsDiv = document.getElementById('charts');
 
-            const summary = await fetch('results_summary.json');
-            const summaryData = await summary.json();
+            const response = await fetch('results.json');
+            const data = await response.json();
             // Format captions
-            const formatTitle = d => (d.target_language_name + "\n" + parseInt(d.speakers / 1_000_00) / 10 + "M speakers\n" + scoreName + ": " + d[scoreKey].toFixed(1))
+            const formatScore = (score) => score > 0 ? score.toFixed(2) : "No benchmark available!"
+            const formatTitle = d => (d.language_name + "\n" + parseInt(d.speakers / 1_000_00) / 10 + "M speakers\n" + scoreName + ": " + formatScore(d[scoreKey]))
 
             // Create summary plot
             const summaryPlot = Plot.plot({
-                width: 800,
+                width: chartsDiv.clientWidth,
                 height: 400,
                 marginBottom: 100,
                 x: { label: "Number of speakers", axis: null },
                 y: { label: `${scoreName} (average across models)` },
                 // color: { scheme: "BrBG" },
                 marks: [
-                    Plot.rectY(summaryData, Plot.stackX({
+                    Plot.rectY(data, Plot.stackX({
                         x: "speakers",
                         order: scoreKey,
                         reverse: true,
                         y2: scoreKey, // y2 to avoid stacking by y
                         title: formatTitle,
                         tip: true,
-                        // fill: d => -d.bleu
+                        fill: d => d[scoreKey] > 0 ? "black" : "pink"
                     })),
-                    Plot.rectY(summaryData, Plot.pointerX(Plot.stackX({
+                    Plot.rectY(data, Plot.pointerX(Plot.stackX({
                         x: "speakers",
                         order: scoreKey,
                         reverse: true,
                         y2: scoreKey, // y2 to avoid stacking by y
                         fill: "grey",
                     }))),
-                    Plot.text(summaryData, Plot.stackX({
+                    Plot.text(data, Plot.stackX({
                         x: "speakers",
                         y2: scoreKey,
                         order: scoreKey,
                         reverse: true,
-                        text: "target_language_name",
+                        text: "language_name",
                         frameAnchor: "bottom",
                         textAnchor: "end",
                         dy: 10,
@@ -87,14 +88,11 @@
             // Add summary plot at the top
             chartsDiv.insertBefore(summaryPlot, chartsDiv.firstChild);
 
-            const response = await fetch('results.json');
-            const results = await response.json();
-
             // Get unique languages with their speaker counts
             const languageMap = new Map();
-            results.forEach(r => {
-                if (!languageMap.has(r.target_language_name)) {
-                    languageMap.set(r.target_language_name, r.speakers);
+            data.forEach(r => {
+                if (!languageMap.has(r.language_name)) {
+                    languageMap.set(r.language_name, r.speakers);
                 }
             });
 
@@ -122,7 +120,7 @@
                 headerDiv.appendChild(speakerP);
                 chartsDiv.appendChild(headerDiv);
 
-                const languageData = results.filter(r => r.target_language_name === language);
+                const languageData = data.filter(r => r.language_name === language)[0]["scores"];
 
                 const descriptor = code => {
                     let [org, model] = code.split("/")
@@ -130,8 +128,7 @@
                 }
 
                 // Plot for how well the models perform on this language
-                if (languageData.length > 2) {
-                    console.log(languageData);
+                if (languageData && languageData.length > 1) {
                     const plot = Plot.plot({
                         width: 400,
                         height: 200,

languagebench.py

@@ -1,7 +1,6 @@
 import asyncio
 import json
 import os
-import random
 from os import getenv
 
 import evaluate
@@ -12,22 +11,19 @@ from dotenv import load_dotenv
 from joblib.memory import Memory
 from openai import AsyncOpenAI
 from tqdm.asyncio import tqdm_asyncio
+from transformers import NllbTokenizer
 
 # config
 models = [
-    "openai/gpt-4o-mini",
+    "openai/gpt-4o",
     "anthropic/claude-3.5-sonnet",
-    "meta-llama/llama-3.1-70b-instruct",  # lots of slow repetitions for LRLs
-    "mistralai/mistral-nemo",
+    "meta-llama/llama-3.1-405b-instruct",  # lots of slow repetitions for LRLs
+    "mistralai/mistral-large",
     # "google/gemini-flash-1.5",  # very fast
     "qwen/qwen-2.5-72b-instruct",  # somewhat slow
 ]
 fast_model = "anthropic/claude-3.5-sonnet"
-original_language = "eng_Latn"
-dataset = "floresp-v2.0-rc.3/dev"
-random.seed(42)
-target_languages = [f.split(".")[1] for f in os.listdir(dataset)]
-detailed_target_languages = random.choices(target_languages, k=5)
+n_sentences = 30
 
 # setup
 load_dotenv()
@@ -36,9 +32,10 @@ client = AsyncOpenAI(
     api_key=getenv("OPENROUTER_API_KEY"),
 )
 cache = Memory(location=".cache", verbose=0).cache
-bleu = evaluate.load("sacrebleu")
+bleu = evaluate.load("bleu")
 bertscore = evaluate.load("bertscore")
-rate_limit = AsyncLimiter(max_rate=15, time_period=1)
+tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
+rate_limit = AsyncLimiter(max_rate=20, time_period=1)
 
 
 def reorder(language_name):
@@ -47,10 +44,65 @@ def reorder(language_name):
     return language_name
 
 
-language_names = pd.read_csv("LanguageCodes.tab", sep="\t")
-language_names["Name"] = language_names["Name"].apply(reorder).str.strip()
-language_stats = pd.read_csv("languages.tsv", sep="\t")
-script_names = pd.read_csv("ScriptCodes.csv")
+# load benchmark languages and scripts
+benchmark_dir = "floresp-v2.0-rc.3/dev"
+benchmark_languages = pd.DataFrame(
+    [f.split(".")[1].split("_", 1) for f in os.listdir(benchmark_dir)],
+    columns=["language_code", "script_code"],
+)
+# hack: drop additional script codes for languages with multiple scripts
+benchmark_languages = benchmark_languages.groupby("language_code").head(1)
+benchmark_languages["in_benchmark"] = True
+
+# load Ethnologue language names
+language_names = (
+    pd.read_csv("LanguageCodes.tab", sep="\t")
+    .rename(columns={"LangID": "language_code", "Name": "language_name"})[
+        ["language_code", "language_name"]
+    ]
+    .assign(language_name=lambda df: df["language_name"].apply(reorder).str.strip())
+)
+
+# load Wikidata speaker stats
+language_stats = (
+    pd.read_csv("languages.tsv", sep="\t")
+    .rename(columns={"iso639_3": "language_code", "maxSpeakers": "speakers"})[
+        ["language_code", "speakers"]
+    ]
+    .dropna(subset=["language_code"])
+)
+language_stats["speakers"] = pd.to_numeric(language_stats["speakers"], errors="coerce")
+ignored_languages = [
+    "zho",  # Chinese -> use Mandarin (cmn) instead
+    "ara",  # Arabic -> use Standard Arabic (arb) instead
+    "pus",  # Pashto -> use Nothern / Central / Southern Pashto instead (pbt / pst / pbu)
+    "fas",  # Persian -> use Iranian Persian (pes) instead
+    "msa",  # Malay -> use Indonesian (ind) instead
+]
+language_stats = language_stats[
+    ~language_stats["language_code"].isin(ignored_languages)
+]
+
+# load unicode script names
+script_names = pd.read_csv("ScriptCodes.csv").rename(
+    columns={"Code": "script_code", "English Name": "script_name"}
+)[["script_code", "script_name"]]
+
+# merge data
+languages = pd.merge(language_stats, language_names, on="language_code", how="outer")
+languages = pd.merge(benchmark_languages, languages, on="language_code", how="outer")
+languages = pd.merge(languages, script_names, on="script_code", how="left")
+languages["in_benchmark"] = languages["in_benchmark"].fillna(False)
+languages = languages.sort_values(by="speakers", ascending=False)
+
+# sample languages to translate from
+original_languages = languages[languages["in_benchmark"]].sample(
+    n=n_sentences, weights="speakers", replace=True, random_state=42
+)
+# sample languages to analyze with all models
+detailed_target_languages = languages[languages["in_benchmark"]].sample(
+    n=25, random_state=42
+)
 
 
 # utils
@@ -94,73 +146,77 @@ async def translate(model, target_language, target_script, sentence):
     return reply.choices[0].message.content
 
 
-def get_language_stats(language_code):
-    lang, script = language_code.split("_", 1)
-    script = script.split("_", 1)[0]
-    stats = language_stats[language_stats["iso639_3"] == lang]
-    if not stats.empty:
-        stats = stats.iloc[0].to_dict()
-    else:
-        stats = dict()
-    stats["script"] = script_names[script_names["Code"] == script]["English Name"].iloc[
-        0
-    ]
-    name_series = language_names[language_names["LangID"] == lang]["Name"]
-    stats["name"] = (
-        name_series.iloc[0]
-        if not name_series.empty
-        else stats.get("itemLabel_en") or stats.get("itemLabel", lang)
-    )
-    return stats
+def mean(l):
+    return sum(l) / len(l) if l else 0
 
 
-def mean(l):
-    return sum(l) / len(l)
+def load_sentences(language):
+    return open(
+        f"{benchmark_dir}/dev.{language.language_code}_{language.script_code}"
+    ).readlines()
 
 
 # evaluation!
 async def main():
-    n = 30
     results = []
-    original_sentences = open(f"{dataset}/dev.{original_language}").readlines()
-    for target_language in target_languages:
-        target_sentences = open(f"{dataset}/dev.{target_language}").readlines()
-        for model in models:
-            if model != fast_model and target_language not in detailed_target_languages:
-                continue
-            stats = get_language_stats(target_language)
-            print(f"{model} -> {stats['name']}")
-            predictions = [
-                translate(model, stats["name"], stats["script"], sentence)
-                for sentence in original_sentences[:n]
-            ]
-            predictions = await tqdm_asyncio.gather(*predictions, miniters=1)
-            metrics = bleu.compute(
-                predictions=predictions,
-                references=target_sentences[:n],
-                tokenize="char",
-            )
-            bert_metrics = bertscore.compute(
-                predictions=predictions,
-                references=target_sentences[:n],
-                model_type="distilbert-base-uncased",
-            )
-            results.append(
-                {
-                    "model": model,
-                    "original_language": original_language,
-                    "target_language": target_language,
-                    "target_language_name": stats["name"],
-                    "speakers": int(stats.get("maxSpeakers", 0)),
-                    "bleu": metrics["score"],
-                    "bert_score": mean(bert_metrics["f1"]),
-                }
-            )
-            with open("results.json", "w") as f:
-                json.dump(results, f, indent=2, ensure_ascii=False)
-            pd.DataFrame(results).groupby("target_language_name").agg(
-                {"bleu": "mean", "bert_score": "mean", "speakers": "mean"}
-            ).reset_index().to_json("results_summary.json", indent=2, orient="records")
+    for language in languages.itertuples():
+        name = (
+            language.language_name
+            if not pd.isna(language.language_name)
+            else language.language_code
+        )
+        print(name)
+        scores = []
+        if language.in_benchmark:
+            target_sentences = load_sentences(language)[:n_sentences]
+            for model in models:
+                if (
+                    model != fast_model
+                    and language.language_code
+                    not in detailed_target_languages.language_code.values
+                ):
+                    continue
+                original_sentences = [
+                    load_sentences(lang)[i]
+                    for i, lang in enumerate(original_languages.itertuples())
+                ]
+                print(model)
+                predictions = [
+                    translate(
+                        model, language.language_name, language.script_name, sentence
+                    )
+                    for sentence in original_sentences
+                ]
+                predictions = await tqdm_asyncio.gather(*predictions, miniters=1)
+                metrics_bleu = bleu.compute(
+                    predictions=predictions,
+                    references=target_sentences,
+                    tokenizer=tokenizer.tokenize,
+                )
+                # metrics_bert = bertscore.compute(
+                #     predictions=predictions,
+                #     references=target_sentences,
+                #     model_type="distilbert-base-uncased",
+                # )
+                scores.append(
+                    {
+                        "model": model,
+                        "bleu": metrics_bleu["bleu"],
+                        # "bert_score": mean(metrics_bert["f1"]),
+                    }
+                )
+        results.append(
+            {
+                "language_name": name,
+                "language_code": language.language_code,
+                "speakers": language.speakers if not pd.isna(language.speakers) else 0,
+                "scores": scores,
+                "bleu": mean([s["bleu"] for s in scores]) or -0.02,
+                # "bert_score": mean([s["bert_score"] for s in scores]),
+            }
+        )
+        with open("results.json", "w") as f:
+            json.dump(results, f, indent=2, ensure_ascii=False)
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -9,9 +9,14 @@ dependencies = [
     "bert-score>=0.3.13",
     "evaluate>=0.4.3",
     "joblib>=1.4.2",
+    "nltk>=3.9.1",
     "openai>=1.52.2",
     "pandas>=2.2.3",
+    "protobuf>=5.28.3",
     "python-dotenv>=1.0.1",
     "sacrebleu>=2.4.3",
+    "sentencepiece>=0.2.0",
+    "tiktoken>=0.8.0",
     "tqdm>=4.66.6",
+    "transformers>=4.46.1",
 ]

results_summary.json

@@ -1,1202 +0,0 @@
-[
-  {
-    "target_language_name":"Aceh",
-    "bleu":39.1659660901,
-    "bert_score":0.8998966595,
-    "speakers":3500032.0
-  },
-  {
-    "target_language_name":"Afrikaans",
-    "bleu":76.8900540777,
-    "bert_score":0.9481831173,
-    "speakers":10300000.0
-  },
-  {
-    "target_language_name":"Amharic",
-    "bleu":43.1544568697,
-    "bert_score":0.989116921,
-    "speakers":25000000.0
-  },
-  {
-    "target_language_name":"Armenian",
-    "bleu":64.6804400806,
-    "bert_score":0.9550812801,
-    "speakers":6700000.0
-  },
-  {
-    "target_language_name":"Assamese",
-    "bleu":47.0351331605,
-    "bert_score":0.928119574,
-    "speakers":15300000.0
-  },
-  {
-    "target_language_name":"Asturian",
-    "bleu":71.3445623493,
-    "bert_score":0.931475842,
-    "speakers":450000.0
-  },
-  {
-    "target_language_name":"Awadhi",
-    "bleu":46.0797144146,
-    "bert_score":0.9333642821,
-    "speakers":22000000.0
-  },
-  {
-    "target_language_name":"Ayacucho Quechua",
-    "bleu":45.6534927028,
-    "bert_score":0.8731370111,
-    "speakers":918200.0
-  },
-  {
-    "target_language_name":"Bali (Indonesia)",
-    "bleu":52.8752419159,
-    "bert_score":0.8934772114,
-    "speakers":4000000.0
-  },
-  {
-    "target_language_name":"Bamanankan",
-    "bleu":38.6939091408,
-    "bert_score":0.8872043769,
-    "speakers":2700000.0
-  },
-  {
-    "target_language_name":"Banjar",
-    "bleu":46.5453977487,
-    "bert_score":0.91599799,
-    "speakers":3500000.0
-  },
-  {
-    "target_language_name":"Bashkort",
-    "bleu":57.5453842927,
-    "bert_score":0.9298217595,
-    "speakers":1200000.0
-  },
-  {
-    "target_language_name":"Basque",
-    "bleu":65.8968721377,
-    "bert_score":0.9192741295,
-    "speakers":750000.0
-  },
-  {
-    "target_language_name":"Belarusian",
-    "bleu":54.5195166442,
-    "bert_score":0.9329862595,
-    "speakers":7900000.0
-  },
-  {
-    "target_language_name":"Bemba",
-    "bleu":47.8068548956,
-    "bert_score":0.889907831,
-    "speakers":3600000.0
-  },
-  {
-    "target_language_name":"Bengali",
-    "bleu":57.1417588816,
-    "bert_score":0.9483523647,
-    "speakers":300000000.0
-  },
-  {
-    "target_language_name":"Bhojpuri",
-    "bleu":44.5412337907,
-    "bert_score":0.9288184981,
-    "speakers":52200000.0
-  },
-  {
-    "target_language_name":"Bokm\u00e5l",
-    "bleu":77.4939513016,
-    "bert_score":0.9550971886,
-    "speakers":4000000.0
-  },
-  {
-    "target_language_name":"Boro (India)",
-    "bleu":36.1100474969,
-    "bert_score":0.925187854,
-    "speakers":1482929.0
-  },
-  {
-    "target_language_name":"Bosnian",
-    "bleu":72.5488027131,
-    "bert_score":0.947693936,
-    "speakers":3500000.0
-  },
-  {
-    "target_language_name":"Bugis",
-    "bleu":44.8388170031,
-    "bert_score":0.8647923966,
-    "speakers":5017800.0
-  },
-  {
-    "target_language_name":"Bulgarian",
-    "bleu":72.9695925131,
-    "bert_score":0.9545443177,
-    "speakers":9000000.0
-  },
-  {
-    "target_language_name":"Burmese",
-    "bleu":55.7235911677,
-    "bert_score":0.9759751062,
-    "speakers":32900000.0
-  },
-  {
-    "target_language_name":"Catalan",
-    "bleu":74.4595007932,
-    "bert_score":0.9464139263,
-    "speakers":5100000.0
-  },
-  {
-    "target_language_name":"Cebuano",
-    "bleu":69.4557958655,
-    "bert_score":0.9321281234,
-    "speakers":15900000.0
-  },
-  {
-    "target_language_name":"Central Aymara",
-    "bleu":42.7698436669,
-    "bert_score":0.8625142018,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Central Kurdish",
-    "bleu":59.1927910692,
-    "bert_score":0.9332568824,
-    "speakers":7250000.0
-  },
-  {
-    "target_language_name":"Central Tibetan",
-    "bleu":51.349075274,
-    "bert_score":0.967157503,
-    "speakers":1200000.0
-  },
-  {
-    "target_language_name":"Chhattisgarhi",
-    "bleu":47.9797501304,
-    "bert_score":0.9363766015,
-    "speakers":16300000.0
-  },
-  {
-    "target_language_name":"Chichewa",
-    "bleu":59.7601680161,
-    "bert_score":0.9069253902,
-    "speakers":12000000.0
-  },
-  {
-    "target_language_name":"Chokwe",
-    "bleu":10.1864074161,
-    "bert_score":0.727788798,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Chuvash",
-    "bleu":45.0546658723,
-    "bert_score":0.9203916192,
-    "speakers":1279650.0
-  },
-  {
-    "target_language_name":"Crimean Tatar",
-    "bleu":52.7050249448,
-    "bert_score":0.8972040812,
-    "speakers":552740.0
-  },
-  {
-    "target_language_name":"Croatian",
-    "bleu":69.5456983662,
-    "bert_score":0.9444877982,
-    "speakers":7000000.0
-  },
-  {
-    "target_language_name":"Czech",
-    "bleu":69.7112290599,
-    "bert_score":0.9384464244,
-    "speakers":10700000.0
-  },
-  {
-    "target_language_name":"Danish",
-    "bleu":78.0935433284,
-    "bert_score":0.9506490747,
-    "speakers":6000000.0
-  },
-  {
-    "target_language_name":"Dari",
-    "bleu":52.5539795795,
-    "bert_score":0.9466466506,
-    "speakers":9600000.0
-  },
-  {
-    "target_language_name":"Dholuo",
-    "bleu":46.4119479071,
-    "bert_score":0.8803233822,
-    "speakers":3000000.0
-  },
-  {
-    "target_language_name":"Dogri",
-    "bleu":44.9153535278,
-    "bert_score":0.934070154,
-    "speakers":2000000.0
-  },
-  {
-    "target_language_name":"Dutch",
-    "bleu":71.1849326315,
-    "bert_score":0.9376831949,
-    "speakers":23100000.0
-  },
-  {
-    "target_language_name":"Dzongkha",
-    "bleu":44.3573814017,
-    "bert_score":0.9664796074,
-    "speakers":237080.0
-  },
-  {
-    "target_language_name":"Eastern Punjabi",
-    "bleu":60.468441109,
-    "bert_score":0.988244007,
-    "speakers":125000000.0
-  },
-  {
-    "target_language_name":"Eastern Yiddish",
-    "bleu":47.5562009325,
-    "bert_score":0.9590989411,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Egyptian Arabic",
-    "bleu":53.6818081038,
-    "bert_score":0.9394114673,
-    "speakers":100542400.0
-  },
-  {
-    "target_language_name":"English",
-    "bleu":75.3501173486,
-    "bert_score":0.8807334363,
-    "speakers":1132366680.0
-  },
-  {
-    "target_language_name":"Esperanto",
-    "bleu":69.6056577554,
-    "bert_score":0.9302131255,
-    "speakers":2000000.0
-  },
-  {
-    "target_language_name":"Faroese",
-    "bleu":65.9147902483,
-    "bert_score":0.9332413753,
-    "speakers":69150.0
-  },
-  {
-    "target_language_name":"Fijian",
-    "bleu":58.2892667246,
-    "bert_score":0.9183188617,
-    "speakers":341270.0
-  },
-  {
-    "target_language_name":"Filipino",
-    "bleu":70.1928498378,
-    "bert_score":0.9269425154,
-    "speakers":90000000.0
-  },
-  {
-    "target_language_name":"Finnish",
-    "bleu":70.9425029518,
-    "bert_score":0.9320579688,
-    "speakers":5413380.0
-  },
-  {
-    "target_language_name":"Fon",
-    "bleu":25.2797773666,
-    "bert_score":0.8664443592,
-    "speakers":1935500.0
-  },
-  {
-    "target_language_name":"French",
-    "bleu":79.3023871219,
-    "bert_score":0.9554367423,
-    "speakers":208157220.0
-  },
-  {
-    "target_language_name":"Friulian",
-    "bleu":66.5488092372,
-    "bert_score":0.9255799611,
-    "speakers":300000.0
-  },
-  {
-    "target_language_name":"Galician",
-    "bleu":68.7024786904,
-    "bert_score":0.9283550183,
-    "speakers":2500000.0
-  },
-  {
-    "target_language_name":"Ganda",
-    "bleu":45.8693322936,
-    "bert_score":0.88344028,
-    "speakers":4100000.0
-  },
-  {
-    "target_language_name":"Georgian",
-    "bleu":61.0166361442,
-    "bert_score":0.9546662311,
-    "speakers":3700000.0
-  },
-  {
-    "target_language_name":"Gikuyu",
-    "bleu":40.9288275291,
-    "bert_score":0.8850945433,
-    "speakers":6623000.0
-  },
-  {
-    "target_language_name":"Goan Konkani",
-    "bleu":47.1084017945,
-    "bert_score":0.9314287245,
-    "speakers":3633900.0
-  },
-  {
-    "target_language_name":"Greek",
-    "bleu":66.2347782153,
-    "bert_score":0.9577525119,
-    "speakers":15000000.0
-  },
-  {
-    "target_language_name":"Gujarati",
-    "bleu":55.5884513452,
-    "bert_score":0.9753397226,
-    "speakers":56400000.0
-  },
-  {
-    "target_language_name":"Haitian Creole",
-    "bleu":63.8532187591,
-    "bert_score":0.93236112,
-    "speakers":9600000.0
-  },
-  {
-    "target_language_name":"Halh Mongolian",
-    "bleu":58.5037789971,
-    "bert_score":0.9380823056,
-    "speakers":2704030.0
-  },
-  {
-    "target_language_name":"Hausa",
-    "bleu":56.3431957901,
-    "bert_score":0.9012877802,
-    "speakers":43900000.0
-  },
-  {
-    "target_language_name":"Hebrew",
-    "bleu":72.0702990513,
-    "bert_score":0.964064618,
-    "speakers":9303950.0
-  },
-  {
-    "target_language_name":"Hindi",
-    "bleu":64.9362166898,
-    "bert_score":0.9463364283,
-    "speakers":341000000.0
-  },
-  {
-    "target_language_name":"Hungarian",
-    "bleu":66.1301119408,
-    "bert_score":0.9249218643,
-    "speakers":12600000.0
-  },
-  {
-    "target_language_name":"Icelandic",
-    "bleu":54.4330055353,
-    "bert_score":0.9120460276,
-    "speakers":358000.0
-  },
-  {
-    "target_language_name":"Igbo",
-    "bleu":46.4017344934,
-    "bert_score":0.9137314638,
-    "speakers":27000000.0
-  },
-  {
-    "target_language_name":"Ilocano",
-    "bleu":62.6058864594,
-    "bert_score":0.9115280092,
-    "speakers":9100000.0
-  },
-  {
-    "target_language_name":"Indonesian",
-    "bleu":72.9087066262,
-    "bert_score":0.9301403503,
-    "speakers":198996550.0
-  },
-  {
-    "target_language_name":"Iranian Persian",
-    "bleu":57.6444169698,
-    "bert_score":0.9476486345,
-    "speakers":52800000.0
-  },
-  {
-    "target_language_name":"Irish",
-    "bleu":69.9725194524,
-    "bert_score":0.9440232972,
-    "speakers":1030000.0
-  },
-  {
-    "target_language_name":"Italian",
-    "bleu":69.1588343572,
-    "bert_score":0.9358606537,
-    "speakers":64819790.0
-  },
-  {
-    "target_language_name":"Japanese",
-    "bleu":49.9166135693,
-    "bert_score":0.9425287286,
-    "speakers":128000000.0
-  },
-  {
-    "target_language_name":"Javanese",
-    "bleu":60.440335299,
-    "bert_score":0.9125308077,
-    "speakers":84308740.0
-  },
-  {
-    "target_language_name":"Jingpho",
-    "bleu":43.5500581403,
-    "bert_score":0.8727998992,
-    "speakers":940000.0
-  },
-  {
-    "target_language_name":"Jula",
-    "bleu":29.5415180297,
-    "bert_score":0.822332112,
-    "speakers":2700000.0
-  },
-  {
-    "target_language_name":"Kabiy\u00e8",
-    "bleu":22.5498504655,
-    "bert_score":0.8587520639,
-    "speakers":1000000.0
-  },
-  {
-    "target_language_name":"Kabuverdianu",
-    "bleu":65.1106010391,
-    "bert_score":0.9213403026,
-    "speakers":871000.0
-  },
-  {
-    "target_language_name":"Kabyle",
-    "bleu":41.1442992587,
-    "bert_score":0.8803219795,
-    "speakers":5586000.0
-  },
-  {
-    "target_language_name":"Kamba",
-    "bleu":41.733489671,
-    "bert_score":0.8780206362,
-    "speakers":3893000.0
-  },
-  {
-    "target_language_name":"Kannada",
-    "bleu":60.0142028332,
-    "bert_score":0.9730932295,
-    "speakers":43600000.0
-  },
-  {
-    "target_language_name":"Kashmiri",
-    "bleu":22.3019416547,
-    "bert_score":0.8984790143,
-    "speakers":6900000.0
-  },
-  {
-    "target_language_name":"Kazakh",
-    "bleu":61.1251621375,
-    "bert_score":0.9379647116,
-    "speakers":13161980.0
-  },
-  {
-    "target_language_name":"Khmer",
-    "bleu":49.2098257043,
-    "bert_score":0.8907732884,
-    "speakers":16600000.0
-  },
-  {
-    "target_language_name":"Kimbundu",
-    "bleu":5.8523457224,
-    "bert_score":0.6849321783,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Kinyarwanda",
-    "bleu":57.2410626756,
-    "bert_score":0.906923449,
-    "speakers":12100000.0
-  },
-  {
-    "target_language_name":"Kituba (Democratic Republic of the Congo)",
-    "bleu":52.8484601602,
-    "bert_score":0.9017938395,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Korean",
-    "bleu":43.6872285974,
-    "bert_score":0.9579092761,
-    "speakers":77300000.0
-  },
-  {
-    "target_language_name":"Kyrgyz",
-    "bleu":57.0824422453,
-    "bert_score":0.9317750076,
-    "speakers":4568480.0
-  },
-  {
-    "target_language_name":"Lao",
-    "bleu":60.0210909677,
-    "bert_score":0.904438438,
-    "speakers":5225552.0
-  },
-  {
-    "target_language_name":"Latgalian",
-    "bleu":56.4843556524,
-    "bert_score":0.9078494012,
-    "speakers":200000.0
-  },
-  {
-    "target_language_name":"Levantine Arabic",
-    "bleu":56.0898634013,
-    "bert_score":0.9437467565,
-    "speakers":44000000.0
-  },
-  {
-    "target_language_name":"Ligurian",
-    "bleu":55.8530636302,
-    "bert_score":0.9047620773,
-    "speakers":500000.0
-  },
-  {
-    "target_language_name":"Limburgish",
-    "bleu":59.4485504982,
-    "bert_score":0.8987095455,
-    "speakers":1600000.0
-  },
-  {
-    "target_language_name":"Lingala",
-    "bleu":30.4322896531,
-    "bert_score":0.8553236572,
-    "speakers":20000000.0
-  },
-  {
-    "target_language_name":"Lithuanian",
-    "bleu":67.1625695571,
-    "bert_score":0.9154702902,
-    "speakers":4000000.0
-  },
-  {
-    "target_language_name":"Lombard",
-    "bleu":46.3884402674,
-    "bert_score":0.8643471499,
-    "speakers":3900000.0
-  },
-  {
-    "target_language_name":"Luba-Kasai",
-    "bleu":45.0655291655,
-    "bert_score":0.8749240279,
-    "speakers":6300000.0
-  },
-  {
-    "target_language_name":"Luxembourgish",
-    "bleu":70.8338190438,
-    "bert_score":0.9297492107,
-    "speakers":391200.0
-  },
-  {
-    "target_language_name":"Macedonian",
-    "bleu":72.2733471437,
-    "bert_score":0.9558346649,
-    "speakers":2000000.0
-  },
-  {
-    "target_language_name":"Magahi",
-    "bleu":58.5474221546,
-    "bert_score":0.9458349566,
-    "speakers":20700000.0
-  },
-  {
-    "target_language_name":"Maithili",
-    "bleu":54.6530071391,
-    "bert_score":0.9433513383,
-    "speakers":33900000.0
-  },
-  {
-    "target_language_name":"Malayalam",
-    "bleu":64.0655894091,
-    "bert_score":0.9803075671,
-    "speakers":37100000.0
-  },
-  {
-    "target_language_name":"Maltese",
-    "bleu":80.0866777263,
-    "bert_score":0.9520254652,
-    "speakers":570000.0
-  },
-  {
-    "target_language_name":"Mandarin Chinese",
-    "bleu":42.5300166785,
-    "bert_score":0.9634857118,
-    "speakers":1074000000.0
-  },
-  {
-    "target_language_name":"Maori",
-    "bleu":54.8319935643,
-    "bert_score":0.9185245017,
-    "speakers":160000.0
-  },
-  {
-    "target_language_name":"Marathi",
-    "bleu":57.4434090711,
-    "bert_score":0.9421781262,
-    "speakers":83100000.0
-  },
-  {
-    "target_language_name":"Meadow Mari",
-    "bleu":49.7911680582,
-    "bert_score":0.9295116961,
-    "speakers":482000.0
-  },
-  {
-    "target_language_name":"Meitei",
-    "bleu":41.2619945571,
-    "bert_score":0.9528288851,
-    "speakers":1470000.0
-  },
-  {
-    "target_language_name":"Merina Malagasy",
-    "bleu":61.0968434546,
-    "bert_score":0.9032936792,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Mesopotamian Arabic",
-    "bleu":49.5184865297,
-    "bert_score":0.9382626355,
-    "speakers":15700000.0
-  },
-  {
-    "target_language_name":"Minangkabau",
-    "bleu":50.7407956197,
-    "bert_score":0.9252789746,
-    "speakers":5530000.0
-  },
-  {
-    "target_language_name":"Mizo",
-    "bleu":51.6558017488,
-    "bert_score":0.8875152906,
-    "speakers":500000.0
-  },
-  {
-    "target_language_name":"Moore",
-    "bleu":32.8458097983,
-    "bert_score":0.8583020627,
-    "speakers":7600000.0
-  },
-  {
-    "target_language_name":"Moroccan Arabic",
-    "bleu":49.3082976781,
-    "bert_score":0.9317501009,
-    "speakers":27500000.0
-  },
-  {
-    "target_language_name":"Najdi Arabic",
-    "bleu":46.4102430377,
-    "bert_score":0.9332984229,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Nepali",
-    "bleu":55.2919347352,
-    "bert_score":0.9358912428,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Nigerian Fulfulde",
-    "bleu":28.1761055913,
-    "bert_score":0.8343587597,
-    "speakers":14500000.0
-  },
-  {
-    "target_language_name":"North Azerbaijani",
-    "bleu":55.5265107063,
-    "bert_score":0.9145456314,
-    "speakers":9220610.0
-  },
-  {
-    "target_language_name":"Northern Kurdish",
-    "bleu":55.7965878227,
-    "bert_score":0.9104436457,
-    "speakers":14600000.0
-  },
-  {
-    "target_language_name":"Northern Sotho",
-    "bleu":62.8769401692,
-    "bert_score":0.9261207898,
-    "speakers":4100000.0
-  },
-  {
-    "target_language_name":"Northern Uzbek",
-    "bleu":63.205573851,
-    "bert_score":0.9120756924,
-    "speakers":26912410.0
-  },
-  {
-    "target_language_name":"Nuer",
-    "bleu":16.5796987951,
-    "bert_score":0.8528214693,
-    "speakers":900000.0
-  },
-  {
-    "target_language_name":"N\u2019Ko",
-    "bleu":32.483490799,
-    "bert_score":0.9823745767,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Occitan",
-    "bleu":71.532740184,
-    "bert_score":0.9337525626,
-    "speakers":542000.0
-  },
-  {
-    "target_language_name":"Odia",
-    "bleu":57.3628096518,
-    "bert_score":0.9768644154,
-    "speakers":34500000.0
-  },
-  {
-    "target_language_name":"Pangasinan",
-    "bleu":56.0048183827,
-    "bert_score":0.8906280657,
-    "speakers":1100000.0
-  },
-  {
-    "target_language_name":"Papiamentu",
-    "bleu":69.7955328133,
-    "bert_score":0.9325902323,
-    "speakers":321300.0
-  },
-  {
-    "target_language_name":"Paraguayan Guaran\u00ed",
-    "bleu":41.7929863707,
-    "bert_score":0.8764786462,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Polish",
-    "bleu":61.8768399674,
-    "bert_score":0.9179250948,
-    "speakers":40200000.0
-  },
-  {
-    "target_language_name":"Portuguese",
-    "bleu":77.4978074222,
-    "bert_score":0.9494876027,
-    "speakers":254300000.0
-  },
-  {
-    "target_language_name":"Romanian",
-    "bleu":76.4907159035,
-    "bert_score":0.9455295324,
-    "speakers":24300000.0
-  },
-  {
-    "target_language_name":"Rundi",
-    "bleu":48.943513629,
-    "bert_score":0.8933652222,
-    "speakers":10800000.0
-  },
-  {
-    "target_language_name":"Russian",
-    "bleu":71.1489441039,
-    "bert_score":0.9518508852,
-    "speakers":171428900.0
-  },
-  {
-    "target_language_name":"Samoan",
-    "bleu":56.7138831423,
-    "bert_score":0.9166683555,
-    "speakers":415720.0
-  },
-  {
-    "target_language_name":"Sango",
-    "bleu":34.8754222657,
-    "bert_score":0.8720244229,
-    "speakers":4600000.0
-  },
-  {
-    "target_language_name":"Sanskrit",
-    "bleu":32.7813249911,
-    "bert_score":0.8987655501,
-    "speakers":49736.0
-  },
-  {
-    "target_language_name":"Santhali",
-    "bleu":31.5119247269,
-    "bert_score":0.944095705,
-    "speakers":7200000.0
-  },
-  {
-    "target_language_name":"Sardinian",
-    "bleu":62.6903914771,
-    "bert_score":0.9118991812,
-    "speakers":1300000.0
-  },
-  {
-    "target_language_name":"Scottish Gaelic",
-    "bleu":62.6044371338,
-    "bert_score":0.9264988482,
-    "speakers":60130.0
-  },
-  {
-    "target_language_name":"Serbian",
-    "bleu":69.9691396176,
-    "bert_score":0.9582955678,
-    "speakers":9000000.0
-  },
-  {
-    "target_language_name":"Setswana",
-    "bleu":55.2288890228,
-    "bert_score":0.9117900888,
-    "speakers":4500000.0
-  },
-  {
-    "target_language_name":"Shan",
-    "bleu":29.2129948577,
-    "bert_score":0.9378574808,
-    "speakers":3000000.0
-  },
-  {
-    "target_language_name":"Shona",
-    "bleu":51.5592191405,
-    "bert_score":0.8798740129,
-    "speakers":9023000.0
-  },
-  {
-    "target_language_name":"Sicilian",
-    "bleu":58.5895359443,
-    "bert_score":0.90428345,
-    "speakers":4700000.0
-  },
-  {
-    "target_language_name":"Silesian",
-    "bleu":56.7836392069,
-    "bert_score":0.9106028736,
-    "speakers":522000.0
-  },
-  {
-    "target_language_name":"Sindhi",
-    "bleu":48.1876056648,
-    "bert_score":0.936702015,
-    "speakers":25000000.0
-  },
-  {
-    "target_language_name":"Sinhala",
-    "bleu":56.7567311796,
-    "bert_score":0.9713358581,
-    "speakers":15300000.0
-  },
-  {
-    "target_language_name":"Slovak",
-    "bleu":67.9284804086,
-    "bert_score":0.9360236605,
-    "speakers":6000000.0
-  },
-  {
-    "target_language_name":"Slovene",
-    "bleu":72.5691270757,
-    "bert_score":0.9432346245,
-    "speakers":2400000.0
-  },
-  {
-    "target_language_name":"Somali",
-    "bleu":55.3706496473,
-    "bert_score":0.908571593,
-    "speakers":16200000.0
-  },
-  {
-    "target_language_name":"South Azerbaijani",
-    "bleu":44.3712804302,
-    "bert_score":0.9420697371,
-    "speakers":15000000.0
-  },
-  {
-    "target_language_name":"Southern Pashto",
-    "bleu":38.3124819374,
-    "bert_score":0.921268179,
-    "speakers":10900000.0
-  },
-  {
-    "target_language_name":"Southern Sotho",
-    "bleu":56.735299554,
-    "bert_score":0.9102749407,
-    "speakers":6000000.0
-  },
-  {
-    "target_language_name":"Southwestern Dinka",
-    "bleu":17.5913281403,
-    "bert_score":0.8016291638,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Spanish",
-    "bleu":63.8467073379,
-    "bert_score":0.9224406302,
-    "speakers":485000000.0
-  },
-  {
-    "target_language_name":"Standard Arabic",
-    "bleu":56.8831262708,
-    "bert_score":0.9168330083,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Standard Estonian",
-    "bleu":67.4156919517,
-    "bert_score":0.9277306815,
-    "speakers":1164770.0
-  },
-  {
-    "target_language_name":"Standard German",
-    "bleu":77.1966515107,
-    "bert_score":0.9468763133,
-    "speakers":105000000.0
-  },
-  {
-    "target_language_name":"Standard Latvian",
-    "bleu":65.0833210037,
-    "bert_score":0.9217625757,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Standard Malay",
-    "bleu":74.2657232798,
-    "bert_score":0.9445500493,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Standard Moroccan Tamazight",
-    "bleu":35.6247648109,
-    "bert_score":0.9847298423,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Sunda",
-    "bleu":56.4065999104,
-    "bert_score":0.9077177823,
-    "speakers":32400000.0
-  },
-  {
-    "target_language_name":"Swahili",
-    "bleu":73.5199042142,
-    "bert_score":0.9450787365,
-    "speakers":82300000.0
-  },
-  {
-    "target_language_name":"Swati",
-    "bleu":52.7746096439,
-    "bert_score":0.8899940272,
-    "speakers":2034200.0
-  },
-  {
-    "target_language_name":"Swedish",
-    "bleu":77.421610247,
-    "bert_score":0.9571870168,
-    "speakers":9244250.0
-  },
-  {
-    "target_language_name":"Tajik",
-    "bleu":60.9783684158,
-    "bert_score":0.9378365338,
-    "speakers":14000000.0
-  },
-  {
-    "target_language_name":"Tamasheq",
-    "bleu":18.4319889721,
-    "bert_score":0.8427422295,
-    "speakers":500000.0
-  },
-  {
-    "target_language_name":"Tamil",
-    "bleu":65.7863221054,
-    "bert_score":0.9536473691,
-    "speakers":75000000.0
-  },
-  {
-    "target_language_name":"Tatar",
-    "bleu":60.3447467213,
-    "bert_score":0.9364115715,
-    "speakers":5427318.0
-  },
-  {
-    "target_language_name":"Ta\u2019izzi-Adeni Arabic",
-    "bleu":49.4139335281,
-    "bert_score":0.9354432185,
-    "speakers":10500000.0
-  },
-  {
-    "target_language_name":"Telugu",
-    "bleu":61.6352457629,
-    "bert_score":0.9790697515,
-    "speakers":82000000.0
-  },
-  {
-    "target_language_name":"Thai",
-    "bleu":62.8125360944,
-    "bert_score":0.9225328485,
-    "speakers":40000000.0
-  },
-  {
-    "target_language_name":"Tigrigna",
-    "bleu":32.8711961703,
-    "bert_score":0.9852415164,
-    "speakers":7507780.0
-  },
-  {
-    "target_language_name":"Tok Pisin",
-    "bleu":56.5407760367,
-    "bert_score":0.9031182428,
-    "speakers":4000000.0
-  },
-  {
-    "target_language_name":"Tosk Albanian",
-    "bleu":69.4218765092,
-    "bert_score":0.9402680953,
-    "speakers":3000000.0
-  },
-  {
-    "target_language_name":"Tsonga",
-    "bleu":58.3516573597,
-    "bert_score":0.9134832978,
-    "speakers":13000000.0
-  },
-  {
-    "target_language_name":"Tumbuka",
-    "bleu":44.0490017392,
-    "bert_score":0.8865564326,
-    "speakers":2680000.0
-  },
-  {
-    "target_language_name":"Tunisian Arabic",
-    "bleu":49.6714090744,
-    "bert_score":0.9337966998,
-    "speakers":11600000.0
-  },
-  {
-    "target_language_name":"Turkish",
-    "bleu":67.1600625676,
-    "bert_score":0.9309494158,
-    "speakers":82231620.0
-  },
-  {
-    "target_language_name":"Turkmen",
-    "bleu":60.5593705936,
-    "bert_score":0.9125106474,
-    "speakers":16000000.0
-  },
-  {
-    "target_language_name":"Twi",
-    "bleu":44.7976562068,
-    "bert_score":0.8913615406,
-    "speakers":3000000.0
-  },
-  {
-    "target_language_name":"Ukrainian",
-    "bleu":68.0976232544,
-    "bert_score":0.9468558848,
-    "speakers":34710100.0
-  },
-  {
-    "target_language_name":"Umbundu",
-    "bleu":21.0802775597,
-    "bert_score":0.8461364289,
-    "speakers":6000000.0
-  },
-  {
-    "target_language_name":"Urdu",
-    "bleu":61.1255457272,
-    "bert_score":0.953888009,
-    "speakers":94022900.0
-  },
-  {
-    "target_language_name":"Uyghur",
-    "bleu":53.5346877103,
-    "bert_score":0.9397906005,
-    "speakers":10400000.0
-  },
-  {
-    "target_language_name":"Venetian",
-    "bleu":60.6140876271,
-    "bert_score":0.9080212533,
-    "speakers":2000000.0
-  },
-  {
-    "target_language_name":"Vietnamese",
-    "bleu":70.3560749464,
-    "bert_score":0.9527418713,
-    "speakers":76000000.0
-  },
-  {
-    "target_language_name":"Waray-Waray",
-    "bleu":66.3850231243,
-    "bert_score":0.920412008,
-    "speakers":3100000.0
-  },
-  {
-    "target_language_name":"Welsh",
-    "bleu":83.3437724474,
-    "bert_score":0.9662299534,
-    "speakers":977366.0
-  },
-  {
-    "target_language_name":"West Central Oromo",
-    "bleu":46.9090350028,
-    "bert_score":0.8845542371,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Wolof",
-    "bleu":42.6430127569,
-    "bert_score":0.8762976408,
-    "speakers":3700000.0
-  },
-  {
-    "target_language_name":"Xhosa",
-    "bleu":55.4688091009,
-    "bert_score":0.9008744816,
-    "speakers":11000000.0
-  },
-  {
-    "target_language_name":"Yerwa Kanuri",
-    "bleu":18.5081787556,
-    "bert_score":0.839997381,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"Yoruba",
-    "bleu":34.2642542268,
-    "bert_score":0.9001545012,
-    "speakers":40000000.0
-  },
-  {
-    "target_language_name":"Yue Chinese",
-    "bleu":34.5614651228,
-    "bert_score":0.9634495397,
-    "speakers":73100000.0
-  },
-  {
-    "target_language_name":"Zulu",
-    "bleu":59.1762078389,
-    "bert_score":0.9099391103,
-    "speakers":15700000.0
-  },
-  {
-    "target_language_name":"nno",
-    "bleu":71.8615646296,
-    "bert_score":0.9335320314,
-    "speakers":0.0
-  },
-  {
-    "target_language_name":"\u00c9w\u00e9",
-    "bleu":41.6614038791,
-    "bert_score":0.8829316159,
-    "speakers":3000000.0
-  }
-]

uv.lock

@@ -253,6 +253,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", size = 49446 },
 ]
 
+[[package]]
+name = "click"
+version = "8.1.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "platform_system == 'Windows'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 },
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -767,11 +779,16 @@ dependencies = [
     { name = "bert-score" },
     { name = "evaluate" },
     { name = "joblib" },
+    { name = "nltk" },
     { name = "openai" },
     { name = "pandas" },
+    { name = "protobuf" },
     { name = "python-dotenv" },
     { name = "sacrebleu" },
+    { name = "sentencepiece" },
+    { name = "tiktoken" },
     { name = "tqdm" },
+    { name = "transformers" },
 ]
 
 [package.metadata]
@@ -780,11 +797,16 @@ requires-dist = [
     { name = "bert-score", specifier = ">=0.3.13" },
     { name = "evaluate", specifier = ">=0.4.3" },
     { name = "joblib", specifier = ">=1.4.2" },
+    { name = "nltk", specifier = ">=3.9.1" },
     { name = "openai", specifier = ">=1.52.2" },
     { name = "pandas", specifier = ">=2.2.3" },
+    { name = "protobuf", specifier = ">=5.28.3" },
     { name = "python-dotenv", specifier = ">=1.0.1" },
     { name = "sacrebleu", specifier = ">=2.4.3" },
+    { name = "sentencepiece", specifier = ">=0.2.0" },
+    { name = "tiktoken", specifier = ">=0.8.0" },
     { name = "tqdm", specifier = ">=4.66.6" },
+    { name = "transformers", specifier = ">=4.46.1" },
 ]
 
 [[package]]
@@ -1083,6 +1105,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263 },
 ]
 
+[[package]]
+name = "nltk"
+version = "3.9.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "joblib" },
+    { name = "regex" },
+    { name = "tqdm" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3c/87/db8be88ad32c2d042420b6fd9ffd4a149f9a0d7f0e86b3f543be2eeeedd2/nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868", size = 2904691 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4d/66/7d9e26593edda06e8cb531874633f7c2372279c3b0f46235539fe546df8b/nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1", size = 1505442 },
+]
+
 [[package]]
 name = "numpy"
 version = "2.1.2"
@@ -1491,6 +1528,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3d/b6/e6d98278f2d49b22b4d033c9f792eda783b9ab2094b041f013fc69bcde87/propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036", size = 11603 },
 ]
 
+[[package]]
+name = "protobuf"
+version = "5.28.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/74/6e/e69eb906fddcb38f8530a12f4b410699972ab7ced4e21524ece9d546ac27/protobuf-5.28.3.tar.gz", hash = "sha256:64badbc49180a5e401f373f9ce7ab1d18b63f7dd4a9cdc43c92b9f0b481cef7b", size = 422479 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/c5/05163fad52d7c43e124a545f1372d18266db36036377ad29de4271134a6a/protobuf-5.28.3-cp310-abi3-win32.whl", hash = "sha256:0c4eec6f987338617072592b97943fdbe30d019c56126493111cf24344c1cc24", size = 419624 },
+    { url = "https://files.pythonhosted.org/packages/9c/4c/4563ebe001ff30dca9d7ed12e471fa098d9759712980cde1fd03a3a44fb7/protobuf-5.28.3-cp310-abi3-win_amd64.whl", hash = "sha256:91fba8f445723fcf400fdbe9ca796b19d3b1242cd873907979b9ed71e4afe868", size = 431464 },
+    { url = "https://files.pythonhosted.org/packages/1c/f2/baf397f3dd1d3e4af7e3f5a0382b868d25ac068eefe1ebde05132333436c/protobuf-5.28.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a3f6857551e53ce35e60b403b8a27b0295f7d6eb63d10484f12bc6879c715687", size = 414743 },
+    { url = "https://files.pythonhosted.org/packages/85/50/cd61a358ba1601f40e7d38bcfba22e053f40ef2c50d55b55926aecc8fec7/protobuf-5.28.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:3fa2de6b8b29d12c61911505d893afe7320ce7ccba4df913e2971461fa36d584", size = 316511 },
+    { url = "https://files.pythonhosted.org/packages/5d/ae/3257b09328c0b4e59535e497b0c7537d4954038bdd53a2f0d2f49d15a7c4/protobuf-5.28.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:712319fbdddb46f21abb66cd33cb9e491a5763b2febd8f228251add221981135", size = 316624 },
+    { url = "https://files.pythonhosted.org/packages/ad/c3/2377c159e28ea89a91cf1ca223f827ae8deccb2c9c401e5ca233cd73002f/protobuf-5.28.3-py3-none-any.whl", hash = "sha256:cee1757663fa32a1ee673434fcf3bf24dd54763c79690201208bafec62f19eed", size = 169511 },
+]
+
 [[package]]
 name = "pyarrow"
 version = "18.0.0"
@@ -1878,6 +1929,38 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/19/46/5d11dc300feaad285c2f1bd784ff3f689f5e0ab6be49aaf568f3a77019eb/safetensors-0.4.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:21742b391b859e67b26c0b2ac37f52c9c0944a879a25ad2f9f9f3cd61e7fda8f", size = 606660 },
 ]
 
+[[package]]
+name = "sentencepiece"
+version = "0.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c9/d2/b9c7ca067c26d8ff085d252c89b5f69609ca93fb85a00ede95f4857865d4/sentencepiece-0.2.0.tar.gz", hash = "sha256:a52c19171daaf2e697dc6cbe67684e0fa341b1248966f6aebb541de654d15843", size = 2632106 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f6/71/98648c3b64b23edb5403f74bcc906ad21766872a6e1ada26ea3f1eb941ab/sentencepiece-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:188779e1298a1c8b8253c7d3ad729cb0a9891e5cef5e5d07ce4592c54869e227", size = 2408979 },
+    { url = "https://files.pythonhosted.org/packages/77/9f/7efbaa6d4c0c718a9affbecc536b03ca62f99f421bdffb531c16030e2d2b/sentencepiece-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bed9cf85b296fa2b76fc2547b9cbb691a523864cebaee86304c43a7b4cb1b452", size = 1238845 },
+    { url = "https://files.pythonhosted.org/packages/1c/e4/c2541027a43ec6962ba9b601805d17ba3f86b38bdeae0e8ac65a2981e248/sentencepiece-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7b67e724bead13f18db6e1d10b6bbdc454af574d70efbb36f27d90387be1ca3", size = 1181472 },
+    { url = "https://files.pythonhosted.org/packages/fd/46/316c1ba6c52b97de76aff7b9da678f7afbb52136afb2987c474d95630e65/sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fde4b08cfe237be4484c6c7c2e2c75fb862cfeab6bd5449ce4caeafd97b767a", size = 1259151 },
+    { url = "https://files.pythonhosted.org/packages/aa/5a/3c48738a0835d76dd06c62b6ac48d39c923cde78dd0f587353bdcbb99851/sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c378492056202d1c48a4979650981635fd97875a00eabb1f00c6a236b013b5e", size = 1355931 },
+    { url = "https://files.pythonhosted.org/packages/a6/27/33019685023221ca8ed98e8ceb7ae5e166032686fa3662c68f1f1edf334e/sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1380ce6540a368de2ef6d7e6ba14ba8f3258df650d39ba7d833b79ee68a52040", size = 1301537 },
+    { url = "https://files.pythonhosted.org/packages/ca/e4/55f97cef14293171fef5f96e96999919ab5b4d1ce95b53547ad653d7e3bf/sentencepiece-0.2.0-cp310-cp310-win32.whl", hash = "sha256:a1151d6a6dd4b43e552394aed0edfe9292820272f0194bd56c7c1660a0c06c3d", size = 936747 },
+    { url = "https://files.pythonhosted.org/packages/85/f4/4ef1a6e0e9dbd8a60780a91df8b7452ada14cfaa0e17b3b8dfa42cecae18/sentencepiece-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:d490142b0521ef22bc1085f061d922a2a6666175bb6b42e588ff95c0db6819b2", size = 991525 },
+    { url = "https://files.pythonhosted.org/packages/32/43/8f8885168a47a02eba1455bd3f4f169f50ad5b8cebd2402d0f5e20854d04/sentencepiece-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:17982700c4f6dbb55fa3594f3d7e5dd1c8659a274af3738e33c987d2a27c9d5c", size = 2409036 },
+    { url = "https://files.pythonhosted.org/packages/0f/35/e63ba28062af0a3d688a9f128e407a1a2608544b2f480cb49bf7f4b1cbb9/sentencepiece-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c867012c0e8bcd5bdad0f791609101cb5c66acb303ab3270218d6debc68a65e", size = 1238921 },
+    { url = "https://files.pythonhosted.org/packages/de/42/ae30952c4a0bd773e90c9bf2579f5533037c886dfc8ec68133d5694f4dd2/sentencepiece-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fd6071249c74f779c5b27183295b9202f8dedb68034e716784364443879eaa6", size = 1181477 },
+    { url = "https://files.pythonhosted.org/packages/e3/ac/2f2ab1d60bb2d795d054eebe5e3f24b164bc21b5a9b75fba7968b3b91b5a/sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f90c55a65013cbb8f4d7aab0599bf925cde4adc67ae43a0d323677b5a1c6cb", size = 1259182 },
+    { url = "https://files.pythonhosted.org/packages/45/fb/14633c6ecf262c468759ffcdb55c3a7ee38fe4eda6a70d75ee7c7d63c58b/sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b293734059ef656dcd65be62ff771507bea8fed0a711b6733976e1ed3add4553", size = 1355537 },
+    { url = "https://files.pythonhosted.org/packages/fb/12/2f5c8d4764b00033cf1c935b702d3bb878d10be9f0b87f0253495832d85f/sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e58b47f933aca74c6a60a79dcb21d5b9e47416256c795c2d58d55cec27f9551d", size = 1301464 },
+    { url = "https://files.pythonhosted.org/packages/4e/b1/67afc0bde24f6dcb3acdea0dd8dcdf4b8b0db240f6bacd39378bd32d09f8/sentencepiece-0.2.0-cp311-cp311-win32.whl", hash = "sha256:c581258cf346b327c62c4f1cebd32691826306f6a41d8c4bec43b010dee08e75", size = 936749 },
+    { url = "https://files.pythonhosted.org/packages/a2/f6/587c62fd21fc988555b85351f50bbde43a51524caafd63bc69240ded14fd/sentencepiece-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:0993dbc665f4113017892f1b87c3904a44d0640eda510abcacdfb07f74286d36", size = 991520 },
+    { url = "https://files.pythonhosted.org/packages/27/5a/141b227ed54293360a9ffbb7bf8252b4e5efc0400cdeac5809340e5d2b21/sentencepiece-0.2.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ea5f536e32ea8ec96086ee00d7a4a131ce583a1b18d130711707c10e69601cb2", size = 2409370 },
+    { url = "https://files.pythonhosted.org/packages/2e/08/a4c135ad6fc2ce26798d14ab72790d66e813efc9589fd30a5316a88ca8d5/sentencepiece-0.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0cb51f53b6aae3c36bafe41e86167c71af8370a039f542c43b0cce5ef24a68c", size = 1239288 },
+    { url = "https://files.pythonhosted.org/packages/49/0a/2fe387f825ac5aad5a0bfe221904882106cac58e1b693ba7818785a882b6/sentencepiece-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3212121805afc58d8b00ab4e7dd1f8f76c203ddb9dc94aa4079618a31cf5da0f", size = 1181597 },
+    { url = "https://files.pythonhosted.org/packages/cc/38/e4698ee2293fe4835dc033c49796a39b3eebd8752098f6bd0aa53a14af1f/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a3149e3066c2a75e0d68a43eb632d7ae728c7925b517f4c05c40f6f7280ce08", size = 1259220 },
+    { url = "https://files.pythonhosted.org/packages/12/24/fd7ef967c9dad2f6e6e5386d0cadaf65cda8b7be6e3861a9ab3121035139/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632f3594d3e7ac8b367bca204cb3fd05a01d5b21455acd097ea4c0e30e2f63d7", size = 1355962 },
+    { url = "https://files.pythonhosted.org/packages/4f/d2/18246f43ca730bb81918f87b7e886531eda32d835811ad9f4657c54eee35/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f295105c6bdbb05bd5e1b0cafbd78ff95036f5d3641e7949455a3f4e5e7c3109", size = 1301706 },
+    { url = "https://files.pythonhosted.org/packages/8a/47/ca237b562f420044ab56ddb4c278672f7e8c866e183730a20e413b38a989/sentencepiece-0.2.0-cp312-cp312-win32.whl", hash = "sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251", size = 936941 },
+    { url = "https://files.pythonhosted.org/packages/c6/97/d159c32642306ee2b70732077632895438867b3b6df282354bd550cf2a67/sentencepiece-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7a673a72aab81fef5ebe755c6e0cc60087d1f3a4700835d40537183c1703a45f", size = 991994 },
+]
+
 [[package]]
 name = "setuptools"
 version = "75.3.0"
@@ -1926,6 +2009,42 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 },
 ]
 
+[[package]]
+name = "tiktoken"
+version = "0.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "regex" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/37/02/576ff3a6639e755c4f70997b2d315f56d6d71e0d046f4fb64cb81a3fb099/tiktoken-0.8.0.tar.gz", hash = "sha256:9ccbb2740f24542534369c5635cfd9b2b3c2490754a78ac8831d99f89f94eeb2", size = 35107 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c9/ba/a35fad753bbca8ba0cc1b0f3402a70256a110ced7ac332cf84ba89fc87ab/tiktoken-0.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b07e33283463089c81ef1467180e3e00ab00d46c2c4bbcef0acab5f771d6695e", size = 1039905 },
+    { url = "https://files.pythonhosted.org/packages/91/05/13dab8fd7460391c387b3e69e14bf1e51ff71fe0a202cd2933cc3ea93fb6/tiktoken-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9269348cb650726f44dd3bbb3f9110ac19a8dcc8f54949ad3ef652ca22a38e21", size = 982417 },
+    { url = "https://files.pythonhosted.org/packages/e9/98/18ec4a8351a6cf4537e40cd6e19a422c10cce1ef00a2fcb716e0a96af58b/tiktoken-0.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e13f37bc4ef2d012731e93e0fef21dc3b7aea5bb9009618de9a4026844e560", size = 1144915 },
+    { url = "https://files.pythonhosted.org/packages/2e/28/cf3633018cbcc6deb7805b700ccd6085c9a5a7f72b38974ee0bffd56d311/tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f13d13c981511331eac0d01a59b5df7c0d4060a8be1e378672822213da51e0a2", size = 1177221 },
+    { url = "https://files.pythonhosted.org/packages/57/81/8a5be305cbd39d4e83a794f9e80c7f2c84b524587b7feb27c797b2046d51/tiktoken-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6b2ddbc79a22621ce8b1166afa9f9a888a664a579350dc7c09346a3b5de837d9", size = 1237398 },
+    { url = "https://files.pythonhosted.org/packages/dc/da/8d1cc3089a83f5cf11c2e489332752981435280285231924557350523a59/tiktoken-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d8c2d0e5ba6453a290b86cd65fc51fedf247e1ba170191715b049dac1f628005", size = 884215 },
+    { url = "https://files.pythonhosted.org/packages/f6/1e/ca48e7bfeeccaf76f3a501bd84db1fa28b3c22c9d1a1f41af9fb7579c5f6/tiktoken-0.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d622d8011e6d6f239297efa42a2657043aaed06c4f68833550cac9e9bc723ef1", size = 1039700 },
+    { url = "https://files.pythonhosted.org/packages/8c/f8/f0101d98d661b34534769c3818f5af631e59c36ac6d07268fbfc89e539ce/tiktoken-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2efaf6199717b4485031b4d6edb94075e4d79177a172f38dd934d911b588d54a", size = 982413 },
+    { url = "https://files.pythonhosted.org/packages/ac/3c/2b95391d9bd520a73830469f80a96e3790e6c0a5ac2444f80f20b4b31051/tiktoken-0.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5637e425ce1fc49cf716d88df3092048359a4b3bbb7da762840426e937ada06d", size = 1144242 },
+    { url = "https://files.pythonhosted.org/packages/01/c4/c4a4360de845217b6aa9709c15773484b50479f36bb50419c443204e5de9/tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fb0e352d1dbe15aba082883058b3cce9e48d33101bdaac1eccf66424feb5b47", size = 1176588 },
+    { url = "https://files.pythonhosted.org/packages/f8/a3/ef984e976822cd6c2227c854f74d2e60cf4cd6fbfca46251199914746f78/tiktoken-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56edfefe896c8f10aba372ab5706b9e3558e78db39dd497c940b47bf228bc419", size = 1237261 },
+    { url = "https://files.pythonhosted.org/packages/1e/86/eea2309dc258fb86c7d9b10db536434fc16420feaa3b6113df18b23db7c2/tiktoken-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:326624128590def898775b722ccc327e90b073714227175ea8febbc920ac0a99", size = 884537 },
+    { url = "https://files.pythonhosted.org/packages/c1/22/34b2e136a6f4af186b6640cbfd6f93400783c9ef6cd550d9eab80628d9de/tiktoken-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:881839cfeae051b3628d9823b2e56b5cc93a9e2efb435f4cf15f17dc45f21586", size = 1039357 },
+    { url = "https://files.pythonhosted.org/packages/04/d2/c793cf49c20f5855fd6ce05d080c0537d7418f22c58e71f392d5e8c8dbf7/tiktoken-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe9399bdc3f29d428f16a2f86c3c8ec20be3eac5f53693ce4980371c3245729b", size = 982616 },
+    { url = "https://files.pythonhosted.org/packages/b3/a1/79846e5ef911cd5d75c844de3fa496a10c91b4b5f550aad695c5df153d72/tiktoken-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a58deb7075d5b69237a3ff4bb51a726670419db6ea62bdcd8bd80c78497d7ab", size = 1144011 },
+    { url = "https://files.pythonhosted.org/packages/26/32/e0e3a859136e95c85a572e4806dc58bf1ddf651108ae8b97d5f3ebe1a244/tiktoken-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2908c0d043a7d03ebd80347266b0e58440bdef5564f84f4d29fb235b5df3b04", size = 1175432 },
+    { url = "https://files.pythonhosted.org/packages/c7/89/926b66e9025b97e9fbabeaa59048a736fe3c3e4530a204109571104f921c/tiktoken-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:294440d21a2a51e12d4238e68a5972095534fe9878be57d905c476017bff99fc", size = 1236576 },
+    { url = "https://files.pythonhosted.org/packages/45/e2/39d4aa02a52bba73b2cd21ba4533c84425ff8786cc63c511d68c8897376e/tiktoken-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d8f3192733ac4d77977432947d563d7e1b310b96497acd3c196c9bddb36ed9db", size = 883824 },
+    { url = "https://files.pythonhosted.org/packages/e3/38/802e79ba0ee5fcbf240cd624143f57744e5d411d2e9d9ad2db70d8395986/tiktoken-0.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:02be1666096aff7da6cbd7cdaa8e7917bfed3467cd64b38b1f112e96d3b06a24", size = 1039648 },
+    { url = "https://files.pythonhosted.org/packages/b1/da/24cdbfc302c98663fbea66f5866f7fa1048405c7564ab88483aea97c3b1a/tiktoken-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94ff53c5c74b535b2cbf431d907fc13c678bbd009ee633a2aca269a04389f9a", size = 982763 },
+    { url = "https://files.pythonhosted.org/packages/e4/f0/0ecf79a279dfa41fc97d00adccf976ecc2556d3c08ef3e25e45eb31f665b/tiktoken-0.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b231f5e8982c245ee3065cd84a4712d64692348bc609d84467c57b4b72dcbc5", size = 1144417 },
+    { url = "https://files.pythonhosted.org/packages/ab/d3/155d2d4514f3471a25dc1d6d20549ef254e2aa9bb5b1060809b1d3b03d3a/tiktoken-0.8.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4177faa809bd55f699e88c96d9bb4635d22e3f59d635ba6fd9ffedf7150b9953", size = 1175108 },
+    { url = "https://files.pythonhosted.org/packages/19/eb/5989e16821ee8300ef8ee13c16effc20dfc26c777d05fbb6825e3c037b81/tiktoken-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5376b6f8dc4753cd81ead935c5f518fa0fbe7e133d9e25f648d8c4dabdd4bad7", size = 1236520 },
+    { url = "https://files.pythonhosted.org/packages/40/59/14b20465f1d1cb89cfbc96ec27e5617b2d41c79da12b5e04e96d689be2a7/tiktoken-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:18228d624807d66c87acd8f25fc135665617cab220671eb65b50f5d70fa51f69", size = 883849 },
+]
+
 [[package]]
 name = "tokenizers"
 version = "0.20.1"