Upload from GitHub Actions: More results

evals/languages.py

@@ -56,4 +56,4 @@ languages = pd.merge(languages, flores, on="bcp_47", how="left")
 languages = pd.merge(languages, fleurs, on="bcp_47", how="left")
 languages = pd.merge(languages, commonvoice, on="bcp_47", how="left")
 languages["in_benchmark"] = languages["bcp_47"].isin(flores["bcp_47"])
-languages = languages.sort_values(by="speakers", ascending=False)
+languages = languages.sort_values(by=["speakers", "language_name"], ascending=False)

evals/main.py

@@ -9,8 +9,8 @@ from tqdm.asyncio import tqdm_asyncio
 # ===== config =====
 
 n_sentences = 10
-n_languages = 18
-n_models = 22
+n_languages = 20
+n_models = 35
 
 # ===== run evaluation and aggregate results =====
 

evals/models.py

@@ -45,6 +45,10 @@ important_models = [
     "amazon/nova-micro-v1",  # 0.09$
 ]
 
+blocklist = [
+    "microsoft/wizardlm-2-8x22b" # temporarily rate-limited
+]
+
 transcription_models = [
     "elevenlabs/scribe_v1",
     "openai/whisper-large-v3",
@@ -200,11 +204,11 @@ def get_cost(row):
 @cache
 def load_models(date: date):
     popular_models = (
-        get_historical_popular_models(date.today())[:30]
+        get_historical_popular_models(date.today())[:20]
         + get_current_popular_models(date.today())[:10]
     )
     popular_models = [m["slug"] for m in popular_models]
-    models = set(important_models + popular_models)
+    models = set(important_models + popular_models) - set(blocklist)
     models = pd.DataFrame(sorted(list(models)), columns=["id"])
     or_metadata = models["id"].apply(get_or_metadata)
     hf_metadata = or_metadata.apply(get_hf_metadata)

languages.json

@@ -319,7 +319,7 @@
     "family":"Indo-European",
     "flores_path":"ita_Latn",
     "fleurs_tag":"it_it",
-    "commonvoice_hours":363.0,
+    "commonvoice_hours":362.0,
     "commonvoice_locale":"it",
     "in_benchmark":true
   },
@@ -864,28 +864,28 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"mag",
+    "bcp_47":"mwr",
     "speakers":15913080,
-    "language_name":"Magahi",
-    "autonym":"Magahi",
+    "language_name":"Marwari",
+    "autonym":"Marwari",
     "family":"Indo-European",
-    "flores_path":"mag_Deva",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
     "commonvoice_locale":null,
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
-    "bcp_47":"mwr",
+    "bcp_47":"mag",
     "speakers":15913080,
-    "language_name":"Marwari",
-    "autonym":"Marwari",
+    "language_name":"Magahi",
+    "autonym":"Magahi",
     "family":"Indo-European",
-    "flores_path":null,
+    "flores_path":"mag_Deva",
     "fleurs_tag":null,
     "commonvoice_hours":null,
     "commonvoice_locale":null,
-    "in_benchmark":false
+    "in_benchmark":true
   },
   {
     "bcp_47":"bgc",
@@ -1183,7 +1183,7 @@
     "family":"Indo-European",
     "flores_path":"bel_Cyrl",
     "fleurs_tag":"be_by",
-    "commonvoice_hours":1808.0,
+    "commonvoice_hours":1807.0,
     "commonvoice_locale":"be",
     "in_benchmark":true
   },
@@ -1195,7 +1195,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"lua_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":2.8,
+    "commonvoice_hours":2.2,
     "commonvoice_locale":"lua",
     "in_benchmark":true
   },
@@ -1955,18 +1955,6 @@
     "commonvoice_locale":"gom",
     "in_benchmark":true
   },
-  {
-    "bcp_47":"kln",
-    "speakers":4068120,
-    "language_name":"Kalenjin",
-    "autonym":"Kalenjin",
-    "family":"Nilotic",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":43.0,
-    "commonvoice_locale":"kln",
-    "in_benchmark":false
-  },
   {
     "bcp_47":"kam",
     "speakers":4068120,
@@ -1979,6 +1967,18 @@
     "commonvoice_locale":"kam",
     "in_benchmark":true
   },
+  {
+    "bcp_47":"kln",
+    "speakers":4068120,
+    "language_name":"Kalenjin",
+    "autonym":"Kalenjin",
+    "family":"Nilotic",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":43.0,
+    "commonvoice_locale":"kln",
+    "in_benchmark":false
+  },
   {
     "bcp_47":"bjn",
     "speakers":4010288,
@@ -2124,10 +2124,10 @@
     "in_benchmark":true
   },
   {
-    "bcp_47":"gbm",
+    "bcp_47":"lmn",
     "speakers":3580443,
-    "language_name":"Garhwali",
-    "autonym":"Garhwali",
+    "language_name":"Lambadi",
+    "autonym":"Lambadi",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2136,10 +2136,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"lmn",
+    "bcp_47":"gbm",
     "speakers":3580443,
-    "language_name":"Lambadi",
-    "autonym":"Lambadi",
+    "language_name":"Garhwali",
+    "autonym":"Garhwali",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2352,27 +2352,27 @@
     "in_benchmark":true
   },
   {
-    "bcp_47":"efi",
+    "bcp_47":"ibb",
     "speakers":2996392,
-    "language_name":"Efik",
-    "autonym":"Efik",
+    "language_name":"Ibibio",
+    "autonym":"Ibibio",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":11.0,
+    "commonvoice_locale":"ibb",
     "in_benchmark":false
   },
   {
-    "bcp_47":"ibb",
+    "bcp_47":"efi",
     "speakers":2996392,
-    "language_name":"Ibibio",
-    "autonym":"Ibibio",
+    "language_name":"Efik",
+    "autonym":"Efik",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":11.0,
-    "commonvoice_locale":"ibb",
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
     "in_benchmark":false
   },
   {
@@ -2484,10 +2484,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"pam",
+    "bcp_47":"fbl",
     "speakers":2511163,
-    "language_name":"Pampanga",
-    "autonym":"Pampanga",
+    "language_name":"West Albay Bikol",
+    "autonym":"West Albay Bikol",
     "family":"Austronesian",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2496,10 +2496,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"fbl",
+    "bcp_47":"pam",
     "speakers":2511163,
-    "language_name":"West Albay Bikol",
-    "autonym":"West Albay Bikol",
+    "language_name":"Pampanga",
+    "autonym":"Pampanga",
     "family":"Austronesian",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2724,10 +2724,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"khn",
+    "bcp_47":"wbr",
     "speakers":1989135,
-    "language_name":"Khandesi",
-    "autonym":"Khandesi",
+    "language_name":"Wagdi",
+    "autonym":"Wagdi",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2748,10 +2748,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"wbr",
+    "bcp_47":"khn",
     "speakers":1989135,
-    "language_name":"Wagdi",
-    "autonym":"Wagdi",
+    "language_name":"Khandesi",
+    "autonym":"Khandesi",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -3535,7 +3535,7 @@
     "family":null,
     "flores_path":"eus_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":380.0,
+    "commonvoice_hours":379.0,
     "commonvoice_locale":"eu",
     "in_benchmark":true
   },
@@ -3559,7 +3559,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":63.0,
+    "commonvoice_hours":62.0,
     "commonvoice_locale":"kbd",
     "in_benchmark":false
   },
@@ -4715,18 +4715,6 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
-  {
-    "bcp_47":"jmc",
-    "speakers":433291,
-    "language_name":"Machame",
-    "autonym":"Kimachame",
-    "family":"Atlantic-Congo",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
-    "in_benchmark":false
-  },
   {
     "bcp_47":"vun",
     "speakers":433291,
@@ -4747,10 +4735,22 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":2.7,
+    "commonvoice_hours":2.5,
     "commonvoice_locale":"rof",
     "in_benchmark":false
   },
+  {
+    "bcp_47":"jmc",
+    "speakers":433291,
+    "language_name":"Machame",
+    "autonym":"Kimachame",
+    "family":"Atlantic-Congo",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
+    "in_benchmark":false
+  },
   {
     "bcp_47":"kjg",
     "speakers":431949,
@@ -5904,10 +5904,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bss",
+    "bcp_47":"kkj",
     "speakers":149823,
-    "language_name":"Akoose",
-    "autonym":"Akoose",
+    "language_name":"Kako",
+    "autonym":"Kakɔ",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
@@ -5916,10 +5916,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"kkj",
+    "bcp_47":"bss",
     "speakers":149823,
-    "language_name":"Kako",
-    "autonym":"Kakɔ",
+    "language_name":"Akoose",
+    "autonym":"Akoose",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
@@ -6240,11 +6240,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"iu",
+    "bcp_47":"pdt",
     "speakers":90466,
-    "language_name":"Inuktitut",
-    "autonym":"Inuktitut",
-    "family":"Eskimo-Aleut",
+    "language_name":"Plautdietsch",
+    "autonym":"Plautdietsch",
+    "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -6252,11 +6252,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"pdt",
+    "bcp_47":"iu",
     "speakers":90466,
-    "language_name":"Plautdietsch",
-    "autonym":"Plautdietsch",
-    "family":"Indo-European",
+    "language_name":"Inuktitut",
+    "autonym":"Inuktitut",
+    "family":"Eskimo-Aleut",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -6367,7 +6367,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":5.4,
+    "commonvoice_hours":4.6,
     "commonvoice_locale":"btv",
     "in_benchmark":false
   },
@@ -7247,6 +7247,18 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
+  {
+    "bcp_47":"gbz",
+    "speakers":7983,
+    "language_name":"Zoroastrian Dari",
+    "autonym":"Zoroastrian Dari",
+    "family":"Indo-European",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
+    "in_benchmark":false
+  },
   {
     "bcp_47":"ik",
     "speakers":7983,
@@ -7260,11 +7272,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"gbz",
-    "speakers":7983,
-    "language_name":"Zoroastrian Dari",
-    "autonym":"Zoroastrian Dari",
-    "family":"Indo-European",
+    "bcp_47":"twq",
+    "speakers":7970,
+    "language_name":"Tasawaq",
+    "autonym":"Tasawaq Senni",
+    "family":"Songhay",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7283,18 +7295,6 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
-  {
-    "bcp_47":"twq",
-    "speakers":7970,
-    "language_name":"Tasawaq",
-    "autonym":"Tasawaq Senni",
-    "family":"Songhay",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
-    "in_benchmark":false
-  },
   {
     "bcp_47":"mic",
     "speakers":7916,
@@ -7567,7 +7567,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":2.4,
+    "commonvoice_hours":2.3,
     "commonvoice_locale":"yav",
     "in_benchmark":false
   },
@@ -7967,6 +7967,18 @@
     "commonvoice_locale":"ie",
     "in_benchmark":false
   },
+  {
+    "bcp_47":"vot",
+    "speakers":0,
+    "language_name":"Votic",
+    "autonym":"Votic",
+    "family":"Uralic",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":0.1,
+    "commonvoice_locale":"vot",
+    "in_benchmark":false
+  },
   {
     "bcp_47":"sgs",
     "speakers":0,
@@ -7991,18 +8003,6 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
-  {
-    "bcp_47":"ann",
-    "speakers":0,
-    "language_name":"Obolo",
-    "autonym":"Obolo",
-    "family":"Atlantic-Congo",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
-    "in_benchmark":false
-  },
   {
     "bcp_47":"pfl",
     "speakers":0,
@@ -8028,11 +8028,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"lzh",
+    "bcp_47":"ann",
     "speakers":0,
-    "language_name":"Literary Chinese",
-    "autonym":"Literary Chinese",
-    "family":"Sino-Tibetan",
+    "language_name":"Obolo",
+    "autonym":"Obolo",
+    "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -8052,11 +8052,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"io",
+    "bcp_47":"lzh",
     "speakers":0,
-    "language_name":"Ido",
-    "autonym":"Ido",
-    "family":"Artificial Language",
+    "language_name":"Literary Chinese",
+    "autonym":"Literary Chinese",
+    "family":"Sino-Tibetan",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -8075,6 +8075,18 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
+  {
+    "bcp_47":"io",
+    "speakers":0,
+    "language_name":"Ido",
+    "autonym":"Ido",
+    "family":"Artificial Language",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
+    "in_benchmark":false
+  },
   {
     "bcp_47":"gez",
     "speakers":0,
@@ -8111,18 +8123,6 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
-  {
-    "bcp_47":"vot",
-    "speakers":0,
-    "language_name":"Votic",
-    "autonym":"Votic",
-    "family":"Uralic",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":0.1,
-    "commonvoice_locale":"vot",
-    "in_benchmark":false
-  },
   {
     "bcp_47":"cad",
     "speakers":0,

models.json

@@ -10,28 +10,6 @@
     "license":null,
     "creation_date":1733356800000
   },
-  {
-    "id":"anthropic\/claude-3-haiku",
-    "name":"Claude 3 Haiku (self-moderated)",
-    "provider_name":"Anthropic",
-    "cost":1.25,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1710288000000
-  },
-  {
-    "id":"cohere\/command-r",
-    "name":"Command R",
-    "provider_name":"Cohere",
-    "cost":1.5,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1710374400000
-  },
   {
     "id":"deepseek\/deepseek-chat",
     "name":"DeepSeek V3",
@@ -131,17 +109,6 @@
     "license":null,
     "creation_date":1727913600000
   },
-  {
-    "id":"google\/gemma-2-9b-it",
-    "name":"Gemma 2 9B",
-    "provider_name":"Google",
-    "cost":0.0,
-    "hf_id":"google\/gemma-2-9b-it",
-    "size":9241705984.0,
-    "type":"Open",
-    "license":"Gemma",
-    "creation_date":1719187200000
-  },
   {
     "id":"google\/gemma-3-27b-it",
     "name":"Gemma 3 27B",
@@ -175,17 +142,6 @@
     "license":"Llama3",
     "creation_date":1713312000000
   },
-  {
-    "id":"meta-llama\/llama-3-8b-instruct",
-    "name":"Llama 3 8B Instruct",
-    "provider_name":"Meta",
-    "cost":0.06,
-    "hf_id":"meta-llama\/Meta-Llama-3-8B-Instruct",
-    "size":8030261248.0,
-    "type":"Open",
-    "license":"Llama3",
-    "creation_date":1713312000000
-  },
   {
     "id":"meta-llama\/llama-3.1-70b-instruct",
     "name":"Llama 3.1 70B Instruct",
@@ -263,28 +219,6 @@
     "license":"Mit",
     "creation_date":1740355200000
   },
-  {
-    "id":"microsoft\/wizardlm-2-8x22b",
-    "name":"WizardLM-2 8x22B",
-    "provider_name":"WizardLM-2 8x22B",
-    "cost":0.5,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1713225600000
-  },
-  {
-    "id":"mistralai\/mistral-7b-instruct",
-    "name":"Mistral 7B Instruct",
-    "provider_name":"Mistral",
-    "cost":0.0,
-    "hf_id":"mistralai\/Mistral-7B-Instruct-v0.3",
-    "size":7248023552.0,
-    "type":"Open",
-    "license":"Apache 2.0",
-    "creation_date":1716336000000
-  },
   {
     "id":"mistralai\/mistral-nemo",
     "name":"Mistral Nemo",
@@ -318,28 +252,6 @@
     "license":"Apache 2.0",
     "creation_date":1741651200000
   },
-  {
-    "id":"mistralai\/mistral-tiny",
-    "name":"Mistral Tiny",
-    "provider_name":"Mistral Tiny",
-    "cost":0.25,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1704844800000
-  },
-  {
-    "id":"nousresearch\/hermes-3-llama-3.1-405b",
-    "name":"Hermes 3 405B Instruct",
-    "provider_name":"Nous",
-    "cost":0.8,
-    "hf_id":"NousResearch\/Hermes-3-Llama-3.1-405B",
-    "size":405853388800.0,
-    "type":"Open",
-    "license":"Llama3",
-    "creation_date":1723507200000
-  },
   {
     "id":"openai\/gpt-4.1-mini",
     "name":"GPT-4.1 Mini",
@@ -372,16 +284,5 @@
     "type":"Commercial",
     "license":null,
     "creation_date":1721260800000
-  },
-  {
-    "id":"openai\/gpt-4o-mini-2024-07-18",
-    "name":"GPT-4o-mini (2024-07-18)",
-    "provider_name":"OpenAI",
-    "cost":0.6,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1721260800000
   }
 ]