Upload from GitHub Actions: fix norwegian

evals/datasets_/arc.py

@@ -1,6 +1,5 @@
 import random
 
-from langcodes import standardize_tag
 from rich import print
 from models import translate_google, get_google_supported_languages
 from tqdm import tqdm
@@ -9,11 +8,11 @@ import asyncio
 from tqdm.asyncio import tqdm_asyncio
 import os
 
-from datasets_.util import _get_dataset_config_names, _load_dataset
+from datasets_.util import _get_dataset_config_names, _load_dataset, standardize_bcp47
 
 slug_uhura_arc_easy = "masakhane/uhura-arc-easy"
 tags_uhura_arc_easy = {
-    standardize_tag(a.split("_")[0], macro=True): a
+    standardize_bcp47(a.split("_")[0]): a
     for a in _get_dataset_config_names(slug_uhura_arc_easy)
     if not a.endswith("unmatched")
 }
@@ -35,7 +34,7 @@ random.shuffle(common_ids_test)
 
 slug_uhura_arc_easy_translated = "fair-forward/arc-easy-autotranslated"
 tags_uhura_arc_easy_translated = {
-    standardize_tag(a.split("_")[0], macro=True): a
+    standardize_bcp47(a.split("_")[0]): a
     for a in _get_dataset_config_names(slug_uhura_arc_easy_translated)
 }
 

evals/datasets_/mgsm.py

@@ -3,8 +3,8 @@ import os
 import random
 
 from datasets import Dataset, load_dataset
-from datasets_.util import _get_dataset_config_names, _load_dataset, cache
-from langcodes import Language, standardize_tag
+from datasets_.util import _get_dataset_config_names, _load_dataset, cache, standardize_bcp47
+from langcodes import Language
 from models import get_google_supported_languages, translate_google
 from rich import print
 from tqdm import tqdm
@@ -12,20 +12,20 @@ from tqdm.asyncio import tqdm_asyncio
 
 slug_mgsm = "juletxara/mgsm"
 tags_mgsm = {
-    standardize_tag(a, macro=True): a for a in _get_dataset_config_names(slug_mgsm)
+    standardize_bcp47(a): a for a in _get_dataset_config_names(slug_mgsm)
 }
 slug_afrimgsm = "masakhane/afrimgsm"
 tags_afrimgsm = {
-    standardize_tag(a, macro=True): a for a in _get_dataset_config_names(slug_afrimgsm)
+    standardize_bcp47(a): a for a in _get_dataset_config_names(slug_afrimgsm)
 }
 slug_gsm8kx = "Eurolingua/gsm8kx"
 tags_gsm8kx = {
-    standardize_tag(a, macro=True): a
+    standardize_bcp47(a): a
     for a in _get_dataset_config_names(slug_gsm8kx, trust_remote_code=True)
 }
 slug_gsm_autotranslated = "fair-forward/gsm-autotranslated"
 tags_gsm_autotranslated = {
-    standardize_tag(a, macro=True): a
+    standardize_bcp47(a): a
     for a in _get_dataset_config_names(slug_gsm_autotranslated)
 }
 

evals/datasets_/mmlu.py

@@ -4,7 +4,7 @@ import random
 from collections import Counter, defaultdict
 
 from datasets import Dataset, load_dataset
-from datasets_.util import _get_dataset_config_names, _load_dataset, cache
+from datasets_.util import _get_dataset_config_names, _load_dataset, cache, standardize_bcp47
 from langcodes import Language, standardize_tag
 from models import get_google_supported_languages, translate_google
 from rich import print
@@ -24,7 +24,7 @@ def print_datasets_analysis():
     ds1 = _load_dataset(slug1, "eng")
     print_counts(slug1, ds1["dev"]["subject"], ds1["test"]["subject"])
     langs1 = _get_dataset_config_names(slug1)
-    langs1 = [standardize_tag(a, macro=True) for a in langs1]
+    langs1 = [standardize_bcp47(a) for a in langs1]
 
     slug2 = "openai/MMMLU"  # does not have dev set! – but: these languages are all also present in Global-MMLU
     ds2 = _load_dataset(slug2, "FR_FR")
@@ -37,7 +37,7 @@ def print_datasets_analysis():
     ds3 = _load_dataset(slug3, "en")
     print_counts(slug3, ds3["dev"]["subject"], ds3["test"]["subject"])
     langs3 = _get_dataset_config_names(slug3)
-    langs3 = [standardize_tag(a, macro=True) for a in langs3]
+    langs3 = [standardize_bcp47(a) for a in langs3]
 
     slug4 = "lighteval/okapi_mmlu"
     ds4 = _load_dataset(slug4, "ar", trust_remote_code=True)
@@ -132,11 +132,11 @@ def add_choices(row):
 
 
 tags_afrimmlu = {
-    standardize_tag(a, macro=True): a
+    standardize_bcp47(a): a
     for a in _get_dataset_config_names("masakhane/afrimmlu")
 }
 tags_global_mmlu = {
-    standardize_tag(a, macro=True): a
+    standardize_bcp47(a): a
     for a in _get_dataset_config_names("CohereForAI/Global-MMLU")
 }
 tags_okapi = _get_dataset_config_names("lighteval/okapi_mmlu")
@@ -145,7 +145,7 @@ tags_mmlux = set(
     for a in _get_dataset_config_names("Eurolingua/mmlux", trust_remote_code=True)
 )
 tags_mmlu_autotranslated = {
-    standardize_tag(a, macro=True): a
+    standardize_bcp47(a): a
     for a in _get_dataset_config_names("fair-forward/mmlu-autotranslated")
 }
 

evals/datasets_/util.py

@@ -6,10 +6,28 @@ from datasets import Dataset, get_dataset_config_names, load_dataset
 from datasets.exceptions import DatasetNotFoundError
 from huggingface_hub.errors import RepositoryNotFoundError
 from joblib.memory import Memory
+from langcodes import standardize_tag
 
 cache = Memory(location=".cache", verbose=0).cache
 TOKEN = os.getenv("HUGGINGFACE_ACCESS_TOKEN")
 
+# Macrolanguage mappings: when standardize_tag returns a macrolanguage,
+# map it to the preferred specific variant for consistency across datasets.
+# This ensures results from different benchmarks use the same language code.
+MACROLANGUAGE_MAPPINGS = {
+    "no": "nb",  # Norwegian -> Norwegian Bokmål (most widely used variant)
+    # Add more mappings here as needed, e.g.:
+    # "ms": "zsm",  # Malay -> Standard Malay
+    # "ar": "arb",  # Arabic -> Standard Arabic
+}
+
+
+def standardize_bcp47(tag: str, macro: bool = True) -> str:
+    """Standardize a BCP-47 tag with consistent macrolanguage handling."""
+    
+    standardized = standardize_tag(tag, macro=macro)
+    return MACROLANGUAGE_MAPPINGS.get(standardized, standardized)
+
 
 @cache
 def _get_dataset_config_names(dataset, **kwargs):