Add number of speakers

index.html

@@ -10,6 +10,17 @@
             padding: 20px;
             font-family: sans-serif;
         }
+
+        .language-header {
+            margin-bottom: 10px;
+        }
+
+        .speaker-count {
+            font-size: 0.8em;
+            color: #666;
+            font-weight: normal;
+            margin: 0;
+        }
     </style>
 </head>
 
@@ -25,15 +36,39 @@
             const response = await fetch('results.json');
             const results = await response.json();
 
-            const languages = [...new Set(results.map(r => r.target_language))];
+            // Get unique languages with their speaker counts
+            const languageMap = new Map();
+            results.forEach(r => {
+                if (!languageMap.has(r.target_language_name)) {
+                    languageMap.set(r.target_language_name, r.speakers);
+                }
+            });
+
+            // Sort languages by speaker count (descending)
+            const languages = [...languageMap.entries()]
+                .sort((a, b) => b[1] - a[1])
+                .map(([lang]) => lang);
+
             const chartsDiv = document.getElementById('charts');
 
             languages.forEach(language => {
+                const headerDiv = document.createElement('div');
+                headerDiv.className = 'language-header';
+
                 const h2 = document.createElement('h2');
                 h2.textContent = language;
-                chartsDiv.appendChild(h2);
+                h2.style.marginBottom = '5px';
+
+                const speakerP = document.createElement('p');
+                speakerP.className = 'speaker-count';
+                const speakerCount = (languageMap.get(language) / 1_000_000).toFixed(1);
+                speakerP.textContent = `${speakerCount}M speakers`;
+
+                headerDiv.appendChild(h2);
+                headerDiv.appendChild(speakerP);
+                chartsDiv.appendChild(headerDiv);
 
-                const languageData = results.filter(r => r.target_language === language);
+                const languageData = results.filter(r => r.target_language_name === language);
 
                 // Create plot using the more idiomatic Observable Plot approach
                 const plot = Plot.plot({

languagebench.py

@@ -4,10 +4,11 @@ import os
 from os import getenv
 
 import evaluate
+import pandas as pd
 from dotenv import load_dotenv
+from joblib.memory import Memory
 from openai import AsyncOpenAI
 from tqdm.asyncio import tqdm_asyncio
-from joblib.memory import Memory
 
 # config
 models = [
@@ -17,9 +18,10 @@ models = [
     "qwen/qwen-2.5-72b-instruct",
     "meta-llama/llama-3.1-8b-instruct",
 ]
+# models = ["gpt-4o-mini"]
 original_language = "eng_Latn"
 dataset = "floresp-v2.0-rc.3/dev"
-# target_languages = [f.split(".")[1] for f in os.listdir(dataset)]
+target_languages = sorted([f.split(".")[1] for f in os.listdir(dataset)][:10])
 target_languages = [
     "eng_Latn",
     "deu_Latn",
@@ -34,9 +36,11 @@ load_dotenv()
 client = AsyncOpenAI(
     base_url="https://openrouter.ai/api/v1",
     api_key=getenv("OPENROUTER_API_KEY"),
+    # api_key=getenv("OPENAI_API_KEY"),
 )
 cache = Memory(location=".cache", verbose=0).cache
 bleu = evaluate.load("sacrebleu")
+language_stats = pd.read_csv("languages.tsv", sep="\t")
 
 
 @cache
@@ -54,6 +58,14 @@ async def translate(model, target_language, sentence):
     return reply.choices[0].message.content
 
 
+def get_language_stats(language_code):
+    lang, script = language_code.split("_")
+    stats = language_stats[language_stats["iso639_3"] == lang]
+    if stats.empty:
+        return dict()
+    return stats.iloc[0].to_dict()
+
+
 async def main():
     n = 30
     results = []
@@ -71,11 +83,14 @@ async def main():
             metrics = bleu.compute(
                 predictions=predictions, references=target_sentences[:n]
             )
+            stats = get_language_stats(target_language)
             results.append(
                 {
                     "model": model,
                     "original_language": original_language,
                     "target_language": target_language,
+                    "target_language_name": stats.get("itemLabel_en", target_language),
+                    "speakers": stats.get("maxSpeakers"),
                     "bleu": metrics["score"],
                 }
             )

languages.rq

@@ -0,0 +1,28 @@
+# https://query.wikidata.org/
+SELECT DISTINCT ?item (MAX(?numberOfSpeakers) AS ?maxSpeakers) (MAX(?speakersTime) AS ?mostRecentTime) ?iso639_1 ?iso639_3 ?itemLabel ?itemLabel_en
+WHERE {
+  ?item wdt:P31 wd:Q34770;          # General "language" type to include all languages
+        wdt:P220 ?iso639_3.          # Language with ISO 639-3 code
+
+  ?item p:P1098 ?numberOfSpeakersStatement.
+  ?numberOfSpeakersStatement ps:P1098 ?numberOfSpeakers.
+
+  OPTIONAL {
+    ?numberOfSpeakersStatement pq:P585 ?speakersTime.
+  }
+
+  OPTIONAL {
+    ?item wdt:P218 ?iso639_1.
+    ?item rdfs:label ?itemLabel_1 . FILTER(lang(?itemLabel_1) = ?iso639_1)
+  }
+  OPTIONAL {
+    ?item rdfs:label ?itemLabel_3 . FILTER(lang(?itemLabel_3) = ?iso639_3)
+  }
+  BIND(COALESCE(?itemLabel_3, ?itemLabel_1) AS ?itemLabel)
+
+  OPTIONAL {
+    ?item rdfs:label ?itemLabel_en . FILTER(lang(?itemLabel_en) = 'en')
+  }
+}
+GROUP BY ?item ?iso639_1 ?iso639_3 ?itemLabel ?itemLabel_en
+ORDER BY DESC(?maxSpeakers) DESC(?mostRecentTime)

results.json

@@ -3,180 +3,240 @@
     "model": "openai/gpt-4o-mini",
     "original_language": "eng_Latn",
     "target_language": "eng_Latn",
+    "target_language_name": "English",
+    "speakers": 1132366680,
     "bleu": 96.0187510193446
   },
   {
     "model": "google/gemini-flash-1.5",
     "original_language": "eng_Latn",
     "target_language": "eng_Latn",
+    "target_language_name": "English",
+    "speakers": 1132366680,
     "bleu": 79.64837722618887
   },
   {
     "model": "anthropic/claude-3.5-sonnet",
     "original_language": "eng_Latn",
     "target_language": "eng_Latn",
+    "target_language_name": "English",
+    "speakers": 1132366680,
     "bleu": 47.89694173473209
   },
   {
     "model": "qwen/qwen-2.5-72b-instruct",
     "original_language": "eng_Latn",
     "target_language": "eng_Latn",
+    "target_language_name": "English",
+    "speakers": 1132366680,
     "bleu": 57.08253125905762
   },
   {
     "model": "meta-llama/llama-3.1-8b-instruct",
     "original_language": "eng_Latn",
     "target_language": "eng_Latn",
+    "target_language_name": "English",
+    "speakers": 1132366680,
     "bleu": 71.3986619616758
   },
   {
     "model": "openai/gpt-4o-mini",
     "original_language": "eng_Latn",
     "target_language": "deu_Latn",
+    "target_language_name": "German",
+    "speakers": 105000000,
     "bleu": 42.76912386979146
   },
   {
     "model": "google/gemini-flash-1.5",
     "original_language": "eng_Latn",
     "target_language": "deu_Latn",
+    "target_language_name": "German",
+    "speakers": 105000000,
     "bleu": 48.166702527508484
   },
   {
     "model": "anthropic/claude-3.5-sonnet",
     "original_language": "eng_Latn",
     "target_language": "deu_Latn",
+    "target_language_name": "German",
+    "speakers": 105000000,
     "bleu": 47.56638188073429
   },
   {
     "model": "qwen/qwen-2.5-72b-instruct",
     "original_language": "eng_Latn",
     "target_language": "deu_Latn",
+    "target_language_name": "German",
+    "speakers": 105000000,
     "bleu": 38.8670415108337
   },
   {
     "model": "meta-llama/llama-3.1-8b-instruct",
     "original_language": "eng_Latn",
     "target_language": "deu_Latn",
+    "target_language_name": "German",
+    "speakers": 105000000,
     "bleu": 32.2942935571844
   },
   {
     "model": "openai/gpt-4o-mini",
     "original_language": "eng_Latn",
     "target_language": "fra_Latn",
+    "target_language_name": "French",
+    "speakers": 208157220,
     "bleu": 47.70220301445618
   },
   {
     "model": "google/gemini-flash-1.5",
     "original_language": "eng_Latn",
     "target_language": "fra_Latn",
+    "target_language_name": "French",
+    "speakers": 208157220,
     "bleu": 49.50529382461407
   },
   {
     "model": "anthropic/claude-3.5-sonnet",
     "original_language": "eng_Latn",
     "target_language": "fra_Latn",
+    "target_language_name": "French",
+    "speakers": 208157220,
     "bleu": 50.55719906730571
   },
   {
     "model": "qwen/qwen-2.5-72b-instruct",
     "original_language": "eng_Latn",
     "target_language": "fra_Latn",
+    "target_language_name": "French",
+    "speakers": 208157220,
     "bleu": 43.43766704709355
   },
   {
     "model": "meta-llama/llama-3.1-8b-instruct",
     "original_language": "eng_Latn",
     "target_language": "fra_Latn",
+    "target_language_name": "French",
+    "speakers": 208157220,
     "bleu": 37.38013101452594
   },
   {
     "model": "openai/gpt-4o-mini",
     "original_language": "eng_Latn",
     "target_language": "spa_Latn",
+    "target_language_name": "Spanish",
+    "speakers": 485000000,
     "bleu": 34.65606074843554
   },
   {
     "model": "google/gemini-flash-1.5",
     "original_language": "eng_Latn",
     "target_language": "spa_Latn",
+    "target_language_name": "Spanish",
+    "speakers": 485000000,
     "bleu": 34.49205632717459
   },
   {
     "model": "anthropic/claude-3.5-sonnet",
     "original_language": "eng_Latn",
     "target_language": "spa_Latn",
+    "target_language_name": "Spanish",
+    "speakers": 485000000,
     "bleu": 34.58637890527096
   },
   {
     "model": "qwen/qwen-2.5-72b-instruct",
     "original_language": "eng_Latn",
     "target_language": "spa_Latn",
+    "target_language_name": "Spanish",
+    "speakers": 485000000,
     "bleu": 33.41419407814188
   },
   {
     "model": "meta-llama/llama-3.1-8b-instruct",
     "original_language": "eng_Latn",
     "target_language": "spa_Latn",
+    "target_language_name": "Spanish",
+    "speakers": 485000000,
     "bleu": 29.470460185415075
   },
   {
     "model": "openai/gpt-4o-mini",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hans",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.7678283495493847
   },
   {
     "model": "google/gemini-flash-1.5",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hans",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.3178534804335777
   },
   {
     "model": "anthropic/claude-3.5-sonnet",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hans",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.8670958769249191
   },
   {
     "model": "qwen/qwen-2.5-72b-instruct",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hans",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.6796400550094367
   },
   {
     "model": "meta-llama/llama-3.1-8b-instruct",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hans",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.027154305073795664
   },
   {
     "model": "openai/gpt-4o-mini",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hant",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 2.175042632198715
   },
   {
     "model": "google/gemini-flash-1.5",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hant",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.3480387797702917
   },
   {
     "model": "anthropic/claude-3.5-sonnet",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hant",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 3.8196828383724886
   },
   {
     "model": "qwen/qwen-2.5-72b-instruct",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hant",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 2.1029807575075994
   },
   {
     "model": "meta-llama/llama-3.1-8b-instruct",
     "original_language": "eng_Latn",
     "target_language": "cmn_Hant",
+    "target_language_name": "Mandarin",
+    "speakers": 1074000000,
     "bleu": 0.017008567925605175
   }
 ]