Add language families

.gitignore

@@ -1,15 +1,10 @@
 floresp-*
+glottolog-*
 LanguageCodes.tab
 ScriptCodes.csv
 .cache
 .env
 
-# Observable
-.DS_Store
-/dist/
-node_modules/
-yarn-error.log
-
 # Python-generated files
 __pycache__/
 *.py[oc]

data/data.txt

@@ -1,4 +1,5 @@
 floresp-v2.0-rc.3: https://github.com/openlanguagedata/flores
+glottolog-5.1: https://github.com/glottolog/glottolog
 languages.csv: generated from https://query.wikidata.org/ using the languages.rq query
 LanguageCodes.tab: https://www.ethnologue.com/
 ScriptCodes.csv: https://www.unicode.org/iso15924/iso15924-codes.html

evals.py

@@ -19,6 +19,7 @@ from requests import get
 from rich import print
 from tqdm.asyncio import tqdm_asyncio
 from transformers import NllbTokenizer
+from pyglottolog import Glottolog
 
 # config
 models = [
@@ -73,6 +74,15 @@ def population(bcp_47):
     return items
 
 
+glottolog = Glottolog("data/glottolog-5.1")
+
+
+@cache
+def language_family(iso_639_3):
+    languoid = glottolog.languoid(iso_639_3)
+    return languoid.family.name if languoid else None
+
+
 def script_name(iso15924):
     return scripts[scripts["iso15924"] == iso15924]["script_name"].values[0]
 
@@ -406,12 +416,8 @@ async def main():
                     "scores": results,
                     "mt_bleu": mean([s["mt_bleu"] for s in results]),
                     "mt_chrf": mean([s["mt_chrf"] for s in results]),
-                    "cls_acc": mean(
-                        [s["cls_acc"] for s in results]
-                    ),
-                    "mlm_chrf": mean(
-                        [s["mlm_chrf"] for s in results]
-                    ),
+                    "cls_acc": mean([s["cls_acc"] for s in results]),
+                    "mlm_chrf": mean([s["mlm_chrf"] for s in results]),
                     "overall_score": mean([s["overall_score"] for s in results]),
                     "commonvoice_hours": language.commonvoice_hours
                     if not pd.isna(language.commonvoice_hours)
@@ -420,6 +426,9 @@ async def main():
                     if not pd.isna(language.commonvoice_locale)
                     else None,
                     "population": population(language.bcp_47),
+                    "language_family": language_family(
+                        language.flores_path.split("_")[0]
+                    ),
                 }
             )
     with open("results.json", "w") as f:

pyproject.toml

@@ -20,6 +20,7 @@ dev-dependencies = [
     "langcodes>=3.5.0",
     "openai>=1.52.2",
     "protobuf>=5.28.3",
+    "pyglottolog>=3.14.0",
     "python-dotenv>=1.0.1",
     "sacrebleu>=2.4.3",
     "sentencepiece>=0.2.0",

results.json

@@ -208,7 +208,8 @@
       "ZA": 17503716,
       "ZM": 2788256,
       "ZW": 6109446
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Chinese",
@@ -252,7 +253,8 @@
       "TW": 22422850,
       "US": 2295209,
       "VN": 1085934
-    }
+    },
+    "language_family": "Sino-Tibetan"
   },
   {
     "language_name": "Hindi",
@@ -282,7 +284,8 @@
       "NP": 127377,
       "UG": 2206,
       "ZA": 1129272
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Spanish",
@@ -345,7 +348,8 @@
       "US": 31933344,
       "UY": 2981097,
       "VE": 23488572
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Arabic",
@@ -407,7 +411,8 @@
       "TN": 10549080,
       "TR": 459298,
       "YE": 22114456
-    }
+    },
+    "language_family": "Afro-Asiatic"
   },
   {
     "language_name": "Urdu",
@@ -436,7 +441,8 @@
       "IN": 66304500,
       "MU": 71727,
       "PK": 221825950
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "French",
@@ -522,7 +528,8 @@
       "VU": 149166,
       "WF": 7610,
       "YT": 110580
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Bangla",
@@ -551,7 +558,8 @@
       "GB": 263044,
       "IN": 107413290,
       "NP": 28508
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Portuguese",
@@ -591,7 +599,8 @@
       "PT": 9890592,
       "ST": 179454,
       "TL": 816395
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Punjabi",
@@ -621,7 +630,8 @@
       "KE": 10170,
       "PK": 163450700,
       "SG": 9314
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Russian",
@@ -668,7 +678,8 @@
       "UA": 20204534,
       "US": 798334,
       "UZ": 4279156
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Swahili",
@@ -701,7 +712,8 @@
       "UG": 32439750,
       "YT": 2716,
       "ZA": 1016
-    }
+    },
+    "language_family": "Atlantic-Congo"
   },
   {
     "language_name": "Indonesian",
@@ -727,7 +739,8 @@
     "population": {
       "ID": 170896640,
       "NL": 311047
-    }
+    },
+    "language_family": "Austronesian"
   },
   {
     "language_name": "German",
@@ -778,7 +791,8 @@
       "SI": 883126,
       "SK": 1196932,
       "US": 1563403
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Japanese",
@@ -805,7 +819,8 @@
       "BR": 444604,
       "CA": 52772,
       "JP": 119231650
-    }
+    },
+    "language_family": "Japonic"
   },
   {
     "language_name": "Telugu",
@@ -830,7 +845,8 @@
     "commonvoice_locale": "te",
     "population": {
       "IN": 95478480
-    }
+    },
+    "language_family": "Dravidian"
   },
   {
     "language_name": "Marathi",
@@ -855,7 +871,8 @@
     "commonvoice_locale": "mr",
     "population": {
       "IN": 92826300
-    }
+    },
+    "language_family": "Indo-European"
   },
   {
     "language_name": "Javanese",
@@ -881,7 +898,8 @@
     "population": {
       "ID": 90788840,
       "MY": 391825
-    }
+    },
+    "language_family": "Austronesian"
   },
   {
     "language_name": "Vietnamese",
@@ -909,6 +927,7 @@
       "CN": 6970,
       "US": 1130973,
       "VN": 84900318
-    }
+    },
+    "language_family": "Austroasiatic"
   }
 ]