Upload from GitHub Actions: improve norwegian fix

evals/countries.py

@@ -26,7 +26,7 @@ def make_country_table(language_table):
                     "name": lang.language_name,
                     "bcp_47": lang.bcp_47,
                     "population": speaker_pop,
-                    "score": lang.average if not pd.isna(lang.average) else 0,
+                    "score": lang.average if not pd.isna(lang.average) else None,
                 }
             )
     for country, languages in countries.items():
@@ -36,7 +36,7 @@ def make_country_table(language_table):
             score = None  # This will make them appear grey on the map
         else:
             score = (
-                sum(entry["score"] * entry["population"] for entry in languages)
+                sum((entry["score"] or 0) * entry["population"] for entry in languages)
                 / speaker_pop
             )
 

evals/datasets_/commonvoice.py

@@ -2,8 +2,8 @@ import re
 from datetime import date
 
 import pandas as pd
+from datasets_.util import standardize_bcp47
 from joblib.memory import Memory
-from langcodes import standardize_tag
 from requests import get
 
 cache = Memory(location=".cache", verbose=0).cache
@@ -23,8 +23,8 @@ commonvoice["bcp_47"] = commonvoice["commonvoice_locale"].apply(
     lambda x: re.sub(r"-[A-Z]{2}$", "", x)
 )
 commonvoice["bcp_47"] = commonvoice["bcp_47"].apply(
-    lambda x: standardize_tag(x, macro=True)
-)  # this does not really seem to get macrolanguages though, e.g. not for Quechua
+    lambda x: standardize_bcp47(x, macro=True)
+)
 commonvoice = (
     commonvoice.groupby("bcp_47")
     .agg({"commonvoice_hours": "sum", "commonvoice_locale": "first"})

evals/datasets_/fleurs.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from langcodes import standardize_tag
+from datasets_.util import standardize_bcp47
 from pathlib import Path
 import tarfile
 import requests
@@ -8,7 +8,7 @@ fleurs_tags = "af_za,am_et,ar_eg,as_in,ast_es,az_az,be_by,bg_bg,bn_in,bs_ba,ca_e
 
 fleurs = pd.DataFrame(fleurs_tags.split(","), columns=["fleurs_tag"])
 fleurs["bcp_47"] = fleurs["fleurs_tag"].apply(
-    lambda x: standardize_tag(x.rsplit("_")[0], macro=True)
+    lambda x: standardize_bcp47(x.rsplit("_")[0], macro=True)
 )
 
 

evals/datasets_/flores.py

@@ -1,8 +1,8 @@
 import re
 
 import pandas as pd
-from datasets_.util import _get_dataset_config_names, _load_dataset
-from langcodes import Language, standardize_tag
+from datasets_.util import _get_dataset_config_names, _load_dataset, standardize_bcp47
+from langcodes import Language
 
 slug = "openlanguagedata/flores_plus"
 splits = _get_dataset_config_names(slug)
@@ -21,7 +21,7 @@ def aggregate_flores_paths(flores_paths):
     if len(flores_paths) == 1:
         return flores_paths.values[0]
     populations = [
-        Language.get(standardize_tag(x, macro=True)).writing_population()
+        Language.get(standardize_bcp47(x, macro=True)).writing_population()
         for x in flores_paths.values
     ]
     return flores_paths.values[populations.index(max(populations))]
@@ -29,7 +29,7 @@ def aggregate_flores_paths(flores_paths):
 
 flores = pd.DataFrame(splits, columns=["flores_path"])
 flores["bcp_47"] = flores["flores_path"].apply(
-    lambda x: standardize_tag(x, macro=True),
+    lambda x: standardize_bcp47(x, macro=True),
 )
 # ignore script (language is language)
 flores["bcp_47"] = flores["bcp_47"].apply(

evals/datasets_/mmlu.py

@@ -192,7 +192,7 @@ async def load_mmlu(language_bcp_47, nr):
     # TODO: add in Okapi, MMLUX @Jonas
     elif language_bcp_47 in tags_mmlu_autotranslated:
         task = _get_mmlu_item(
-            "fair-forward/mmlu-autotranslated", language_bcp_47, category, nr
+            "fair-forward/mmlu-autotranslated", tags_mmlu_autotranslated[language_bcp_47], category, nr
         )
         return (
             "fair-forward/mmlu-autotranslated",

evals/datasets_/truthfulqa.py

@@ -1,7 +1,7 @@
 import random
 from collections import Counter, defaultdict
 
-from langcodes import Language, standardize_tag
+from langcodes import Language
 from rich import print
 from tqdm import tqdm
 import asyncio
@@ -11,19 +11,19 @@ import os
 from datasets import Dataset, load_dataset
 from models import translate_google, get_google_supported_languages
 
-from datasets_.util import _get_dataset_config_names, _load_dataset
+from datasets_.util import _get_dataset_config_names, _load_dataset, standardize_bcp47
 
 slug_uhura_truthfulqa = "masakhane/uhura-truthfulqa"
 slug_truthfulqa_autotranslated = "fair-forward/truthfulqa-autotranslated"
 
 tags_uhura_truthfulqa = {
-    standardize_tag(a.split("_")[0], macro=True): a
+    standardize_bcp47(a.split("_")[0], macro=True): a
     for a in _get_dataset_config_names(slug_uhura_truthfulqa)
     if a.endswith("multiple_choice")
 }
 
 tags_truthfulqa_autotranslated = {
-    standardize_tag(a, macro=True): a
+    standardize_bcp47(a, macro=True): a
     for a in _get_dataset_config_names(slug_truthfulqa_autotranslated)
 }
 tags_truthfulqa_autotranslated = {}

evals/datasets_/util.py

@@ -16,9 +16,11 @@ TOKEN = os.getenv("HUGGINGFACE_ACCESS_TOKEN")
 # This ensures results from different benchmarks use the same language code.
 MACROLANGUAGE_MAPPINGS = {
     "no": "nb",  # Norwegian -> Norwegian Bokmål (most widely used variant)
-    # Add more mappings here as needed, e.g.:
-    # "ms": "zsm",  # Malay -> Standard Malay
-    # "ar": "arb",  # Arabic -> Standard Arabic
+    # Add more mappings here if they cause duplicate entries in the languages table:
+    # "ms": "zsm",  # Malay -> Standard Malay (if both appear in population data)
+    # "ar": "arb",  # Arabic -> Standard Arabic (if both appear in population data)
+    # "zh": "cmn",  # Chinese -> Mandarin Chinese (if both appear in population data)
+    # Check LANGUAGE_SPEAKING_POPULATION to see which macrolanguages need mapping
 }
 
 

evals/languages.py

@@ -4,8 +4,9 @@ import pandas as pd
 from datasets_.commonvoice import commonvoice
 from datasets_.fleurs import fleurs
 from datasets_.flores import flores
+from datasets_.util import standardize_bcp47
 from joblib.memory import Memory
-from langcodes import Language, standardize_tag
+from langcodes import Language
 from language_data.population_data import LANGUAGE_SPEAKING_POPULATION
 
 cache = Memory(location=".cache", verbose=0).cache
@@ -17,6 +18,10 @@ languages = {
     if not re.match(r".*-[A-Z]{2}$", lang)
 }
 languages = pd.DataFrame(list(languages.items()), columns=["bcp_47", "speakers"])
+# Standardize language codes to avoid duplicates (e.g., "no" and "nb" for Norwegian)
+languages["bcp_47"] = languages["bcp_47"].apply(standardize_bcp47)
+# Merge any duplicates by summing speakers
+languages = languages.groupby("bcp_47", as_index=False).agg({"speakers": "sum"})
 languages["language_name"] = languages["bcp_47"].apply(
     lambda x: Language.get(x).display_name()
 )
@@ -28,7 +33,7 @@ glottolog = pd.read_csv(
     "data/glottolog_languoid.csv/languoid.csv", na_values=[""], keep_default_na=False
 )  # Min _Nan_ Chinese is not N/A!
 glottolog["bcp_47"] = glottolog["iso639P3code"].apply(
-    lambda x: standardize_tag(x, macro=True) if not pd.isna(x) else None
+    lambda x: standardize_bcp47(x, macro=True) if not pd.isna(x) else None
 )
 
 

evals/models.py

@@ -26,10 +26,10 @@ important_models = [
     "openai/gpt-4o",  # 10$
     "openai/gpt-3.5-turbo", # $1.50
     "openai/gpt-oss-120b",
-    "anthropic/claude-4.5-sonnet",
-    "anthropic/claude-4.5-haiku",
+    "anthropic/claude-sonnet-4.5",
+    "anthropic/claude-haiku-4.5",
     "anthropic/claude-opus-4.1",  # 15$
-    "anthropic/claude-4-sonnet",
+    "anthropic/claude-sonnet-4",
     "anthropic/claude-3.7-sonnet",  # 15$
     "anthropic/claude-3.5-sonnet",
     "mistralai/mistral-small-3.2-24b-instruct",  # 0.3$
@@ -41,6 +41,7 @@ important_models = [
     "google/gemini-2.5-flash-lite",  # 0.3$
     "google/gemma-3-27b-it",  # 0.2$
     # "x-ai/grok-4", # $15
+    "x-ai/grok-4-fast", 
     # "x-ai/grok-3", # $15
     "cohere/command-a",
     "qwen/qwen3-32b",
@@ -52,9 +53,13 @@ important_models = [
     # "qwen/qwen-2-72b-instruct",  # 0.9$
     "deepseek/deepseek-v3.2-exp",
     "microsoft/phi-4",  # 0.07$
+    "amazon/nova-premier-v1", # 12.5$
     "amazon/nova-pro-v1",  # 0.09$
     "moonshotai/kimi-k2",  # 0.6$
+    # "moonshotai/kimi-k2-thinking", # 2.5$
     "baidu/ernie-4.5-300b-a47b",
+    # "baidu/ernie-4.5-21b-a3b-thinking",
+    "z-ai/glm-4.6", # 1.75$
 ]
 
 blocklist = [

frontend/package-lock.json

@@ -73,6 +73,7 @@
       "integrity": "sha512-2BCOP7TN8M+gVDj7/ht3hsaO/B/n5oDbiAyyvnRlNOs+u1o+JWNYTQrmpuNp1/Wq2gcFrI01JAW+paEKDMx/CA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/code-frame": "^7.27.1",
         "@babel/generator": "^7.28.3",
@@ -767,6 +768,7 @@
       "integrity": "sha512-p9OkPbZ5G7UT1MofwYFigGebnrzGJacoBSQM0/6bi/PUMVE+qlWDD/OalvQKbwgQzU6dl0xAv6r4X7Jme0RYxA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.27.1"
       },
@@ -1708,6 +1710,7 @@
       "integrity": "sha512-2KH4LWGSrJIkVf5tSiBFYuXDAoWRq2MMwgivCf+93dd0GQi8RXLjKA/0EvRnVV5G0hrHczsquXuD01L8s6dmBw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/helper-annotate-as-pure": "^7.27.1",
         "@babel/helper-module-imports": "^7.27.1",
@@ -3667,6 +3670,7 @@
       "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz",
       "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/code-frame": "^7.10.4",
         "@babel/runtime": "^7.12.5",
@@ -4047,8 +4051,7 @@
       "version": "15.7.15",
       "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz",
       "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==",
-      "license": "MIT",
-      "peer": true
+      "license": "MIT"
     },
     "node_modules/@types/q": {
       "version": "1.5.8",
@@ -4214,6 +4217,7 @@
       "integrity": "sha512-TiZzBSJja/LbhNPvk6yc0JrX9XqhQ0hdh6M2svYfsHGejaKFIAGd9MQ+ERIMzLGlN/kZoYIgdxFV0PuljTKXag==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@eslint-community/regexpp": "^4.4.0",
         "@typescript-eslint/scope-manager": "5.62.0",
@@ -4269,6 +4273,7 @@
       "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
       "dev": true,
       "license": "BSD-2-Clause",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/scope-manager": "5.62.0",
         "@typescript-eslint/types": "5.62.0",
@@ -4668,6 +4673,7 @@
       "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -4775,6 +4781,7 @@
       "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "fast-json-stable-stringify": "^2.0.0",
@@ -5744,6 +5751,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "caniuse-lite": "^1.0.30001737",
         "electron-to-chromium": "^1.5.211",
@@ -7204,6 +7212,7 @@
       "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
       "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
       "license": "ISC",
+      "peer": true,
       "engines": {
         "node": ">=12"
       }
@@ -8177,6 +8186,7 @@
       "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.6.1",
@@ -11195,6 +11205,7 @@
       "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@jest/core": "^27.5.1",
         "import-local": "^3.0.2",
@@ -13635,6 +13646,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "nanoid": "^3.3.11",
         "picocolors": "^1.1.1",
@@ -14893,6 +14905,7 @@
       "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "cssesc": "^3.0.0",
         "util-deprecate": "^1.0.2"
@@ -15286,6 +15299,7 @@
       "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
       "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "loose-envify": "^1.1.0"
       },
@@ -15427,6 +15441,7 @@
       "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
       "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "loose-envify": "^1.1.0",
         "scheduler": "^0.23.2"
@@ -15454,6 +15469,7 @@
       "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -15967,6 +15983,7 @@
       "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "bin": {
         "rollup": "dist/bin/rollup"
       },
@@ -16232,6 +16249,7 @@
       "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
         "fast-uri": "^3.0.1",
@@ -18039,6 +18057,7 @@
       "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
       "dev": true,
       "license": "(MIT OR CC0-1.0)",
+      "peer": true,
       "engines": {
         "node": ">=10"
       },
@@ -18505,6 +18524,7 @@
       "integrity": "sha512-7b0dTKR3Ed//AD/6kkx/o7duS8H3f1a4w3BYpIriX4BzIhjkn4teo05cptsxvLesHFKK5KObnadmCHBwGc+51A==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/eslint-scope": "^3.7.7",
         "@types/estree": "^1.0.8",
@@ -18578,6 +18598,7 @@
       "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/bonjour": "^3.5.9",
         "@types/connect-history-api-fallback": "^1.3.5",
@@ -19015,6 +19036,7 @@
       "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
         "fast-uri": "^3.0.1",

frontend/src/components/WorldMap.js

@@ -26,10 +26,10 @@ const makeTitle = data => d => {
         a =>
           `${smoothProgressBar(a.population / pop)} ${
             a.name
-          } – ${a.score === null || a.score === undefined ? "n/a" : a.score.toFixed(2)}`
+          } – ${a.score === null || a.score === undefined ? "na" : a.score.toFixed(2)}`
       )
       .join('\n\n') + (languages?.length > 10 ? `\n\n...` : '')
-  return `${d.properties.ADMIN} – ${cData?.score === null || cData?.score === undefined ? "n/a" : cData.score.toFixed(2)}\n\n${langstring}`
+  return `${d.properties.ADMIN} – ${cData?.score === null || cData?.score === undefined ? "na" : cData.score.toFixed(2)}\n\n${langstring}`
 }
 
 const WorldMap = ({ data, width = 750, height = 500, allLanguages = [] }) => {