Upload from GitHub Actions: New results

evals/download_data.py

@@ -15,7 +15,7 @@ from datasets_.fleurs import fleurs
 
 
 # Add project root to sys.path (still useful for potential future imports if needed)
-project_root = Path(__file__).resolve().parent
+project_root = Path(__file__).resolve().parent.parent
 if str(project_root) not in sys.path:
     sys.path.append(str(project_root))
 

evals/languages.py

@@ -1,9 +1,9 @@
 import re
 
 import pandas as pd
-from .datasets_.commonvoice import commonvoice
-from .datasets_.fleurs import fleurs
-from .datasets_.flores import flores
+from datasets_.commonvoice import commonvoice
+from datasets_.fleurs import fleurs
+from datasets_.flores import flores
 from joblib.memory import Memory
 from langcodes import Language, standardize_tag
 from language_data.population_data import LANGUAGE_SPEAKING_POPULATION

evals/main.py

@@ -1,11 +1,10 @@
 import asyncio
 
 import pandas as pd
-from tqdm.asyncio import tqdm_asyncio
-
 from languages import languages
 from models import models
 from tasks import tasks
+from tqdm.asyncio import tqdm_asyncio
 
 # ===== config =====
 
@@ -17,32 +16,44 @@ n_models = 25
 
 
 async def evaluate():
+    # save up-to-date info on models and languages
+    args = dict(orient="records", indent=2, force_ascii=False)
+    pd.DataFrame(models).to_json("models.json", **args)
+    pd.DataFrame(languages).to_json("languages.json", **args)
     print("running evaluations")
     old_results = pd.read_json("results.json")
-    results = [
-        task(model, lang.bcp_47, i)
+    # get all combinations of model, language and task
+    combis = [
+        (model, lang.bcp_47, task_name)
         for task_name, task in tasks.items()
-        for i in range(n_sentences)
         for lang in languages.iloc[:n_languages].itertuples()
         for model in models["id"].iloc[:n_models]
-        if len(
-            old_results[
-                (old_results["model"] == model)
-                & (old_results["bcp_47"] == lang.bcp_47)
-                & (old_results["task"] == task_name)
-                & (old_results["sentence_nr"] == i)
-            ]
-        )
-        == 0
+    ]
+    # filter out combinations that have already been evaluated
+    combis = pd.DataFrame(combis, columns=["model", "bcp_47", "task"])
+    combis = combis.merge(old_results, on=["model", "bcp_47", "task"], how="left")
+    combis = combis[combis["metric"].isna()][["model", "bcp_47", "task"]]
+    print(combis["model"].unique())
+    # run evaluations
+    results = [
+        tasks[task_name](model, bcp_47, i)
+        for i in range(n_sentences)
+        for model, bcp_47, task_name in combis.itertuples(index=False)
     ]
     results = await tqdm_asyncio.gather(*results, miniters=1)
     results = [r for group in results for r in group]
-    results = pd.DataFrame(results)
-    results = pd.concat([old_results, results])
-    args = dict(orient="records", indent=2, force_ascii=False)
-    results.to_json("results.json", **args)
-    pd.DataFrame(models).to_json("models.json", **args)
-    pd.DataFrame(languages).to_json("languages.json", **args)
+    if results:
+        # aggregate results
+        results = pd.DataFrame(results)
+        results = (
+            results.groupby(["model", "bcp_47", "task", "metric"])
+            .agg({"score": "mean"})
+            .reset_index()
+        )
+        # save results
+        results = pd.concat([old_results, results])
+        results = results.sort_values(by=["model", "bcp_47", "task", "metric"])
+        results.to_json("results.json", **args)
 
 
 if __name__ == "__main__":

evals/models.py

@@ -15,7 +15,7 @@ from requests import HTTPError, get
 
 # for development purposes, all languages will be evaluated on the fast models
 # and only a sample of languages will be evaluated on all models
-models = [
+important_models = [
     "meta-llama/llama-4-maverick",  # 0.6$
     "meta-llama/llama-3.3-70b-instruct",  # 0.3$
     "meta-llama/llama-3.1-70b-instruct",  # 0.3$
@@ -24,8 +24,8 @@ models = [
     "openai/gpt-4.1-mini",  # 1.6$
     "openai/gpt-4.1-nano",  # 0.4$
     "openai/gpt-4o-mini",  # 0.6$
-    "openai/gpt-3.5-turbo-0613",  # 2$
-    "openai/gpt-3.5-turbo",  # 1.5$
+    # "openai/gpt-3.5-turbo-0613",  # 2$
+    # "openai/gpt-3.5-turbo",  # 1.5$
     # "anthropic/claude-3.5-haiku", # 4$ -> too expensive for dev
     "mistralai/mistral-small-3.1-24b-instruct",  # 0.3$
     "mistralai/mistral-saba",  # 0.6$
@@ -65,8 +65,10 @@ def get_models(date: date):
 
 def get_model(permaslug):
     models = get_models(date.today())
-    slugs = [m for m in models if m["permaslug"] == permaslug]
-    return slugs[0] if len(slugs) == 1 else None
+    slugs = [m for m in models if m["permaslug"] == permaslug and m["endpoint"] and not m["endpoint"]["is_free"]]
+    if len(slugs) == 0:
+        print(f"no model found for {permaslug}")
+    return slugs[0] if len(slugs) >= 1 else None
 
 
 @cache
@@ -81,7 +83,8 @@ def get_historical_popular_models(date: date):
                 continue
             counts[model.split(":")[0]] += count
     counts = sorted(counts.items(), key=lambda x: x[1], reverse=True)
-    return [get_model(model) for model, _ in counts]
+    models = [get_model(model) for model, _ in counts]
+    return [m for m in models if m]
 
 
 @cache
@@ -90,23 +93,10 @@ def get_current_popular_models(date: date):
     data = re.search(r'{\\"rankMap\\":(.*)\}\]\\n"\]\)</script>', raw).group(1)
     data = json.loads(data.replace("\\", ""))["day"]
     data = sorted(data, key=lambda x: x["total_prompt_tokens"], reverse=True)
-    return [get_model(model["model_permaslug"]) for model in data]
+    models = [get_model(model["model_permaslug"]) for model in data]
+    return [m for m in models if m]
 
 
-popular_models = (
-    get_historical_popular_models(date.today())[:5]
-    + get_current_popular_models(date.today())[:5]
-)
-popular_models = [get_model(m) for m in popular_models if get_model(m)]
-popular_models = [
-    m for m in popular_models if m["endpoint"] and not m["endpoint"]["is_free"]
-]
-popular_models = [m["slug"] for m in popular_models]
-popular_models = [
-    m for m in popular_models if m and m not in models and m not in blocklist
-]
-models += popular_models
-
 load_dotenv()
 client = AsyncOpenAI(
     base_url="https://openrouter.ai/api/v1",
@@ -158,9 +148,6 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
             raise ValueError(f"Model {model} not supported")
 
 
-models = pd.DataFrame(models, columns=["id"])
-
-
 def get_or_metadata(id):
     # get metadata from OpenRouter
     models = get_models(date.today())
@@ -210,21 +197,34 @@ def get_cost(row):
     return round(cost * 1_000_000, 2)
 
 
-or_metadata = models["id"].apply(get_or_metadata)
-hf_metadata = or_metadata.apply(get_hf_metadata)
-creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
-creation_date_or = pd.to_datetime(
-    or_metadata.str["created_at"].str.split("T").str[0]
-).dt.date
-
-models = models.assign(
-    name=or_metadata.str["short_name"],
-    provider_name=or_metadata.str["name"].str.split(": ").str[0],
-    cost=or_metadata.apply(get_cost),
-    hf_id=hf_metadata.str["hf_id"],
-    size=hf_metadata.str["size"],
-    type=hf_metadata.str["type"],
-    license=hf_metadata.str["license"],
-    creation_date=creation_date_hf.combine_first(creation_date_or),
-)
-models = models[models["cost"] <= 2.0].reset_index(drop=True)
+@cache
+def load_models(date: date):
+    popular_models = (
+        get_historical_popular_models(date.today())[:10]
+        + get_current_popular_models(date.today())[:10]
+    )
+    popular_models = [m["slug"] for m in popular_models]
+    models = set(important_models + popular_models) - set(blocklist)
+    models = pd.DataFrame(sorted(list(models)), columns=["id"])
+    or_metadata = models["id"].apply(get_or_metadata)
+    hf_metadata = or_metadata.apply(get_hf_metadata)
+    creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
+    creation_date_or = pd.to_datetime(
+        or_metadata.str["created_at"].str.split("T").str[0]
+    ).dt.date
+
+    models = models.assign(
+        name=or_metadata.str["short_name"],
+        provider_name=or_metadata.str["name"].str.split(": ").str[0],
+        cost=or_metadata.apply(get_cost),
+        hf_id=hf_metadata.str["hf_id"],
+        size=hf_metadata.str["size"],
+        type=hf_metadata.str["type"],
+        license=hf_metadata.str["license"],
+        creation_date=creation_date_hf.combine_first(creation_date_or),
+    )
+    models = models[models["cost"] <= 2.0].reset_index(drop=True)
+    return models
+
+
+models = load_models(date.today())

languages.json

@@ -163,7 +163,7 @@
     "family":"Indo-European",
     "flores_path":"deu_Latn",
     "fleurs_tag":"de_de",
-    "commonvoice_hours":1362.0,
+    "commonvoice_hours":1363.0,
     "commonvoice_locale":"de",
     "in_benchmark":true
   },
@@ -269,11 +269,11 @@
     "language_name":"Wu Chinese",
     "autonym":"Wu Chinese",
     "family":"Sino-Tibetan",
-    "flores_path":"wuu_Hans",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
     "commonvoice_locale":null,
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"tr",
@@ -485,11 +485,11 @@
     "language_name":"North Levantine Arabic",
     "autonym":"العامية",
     "family":"Afro-Asiatic",
-    "flores_path":"apc_Arab",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
     "commonvoice_locale":null,
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"ms",
@@ -619,7 +619,7 @@
     "family":"Indo-European",
     "flores_path":"nld_Latn",
     "fleurs_tag":"nl_nl",
-    "commonvoice_hours":119.0,
+    "commonvoice_hours":120.0,
     "commonvoice_locale":"nl",
     "in_benchmark":true
   },
@@ -876,10 +876,10 @@
     "in_benchmark":true
   },
   {
-    "bcp_47":"mwr",
+    "bcp_47":"bgc",
     "speakers":15913080,
-    "language_name":"Marwari",
-    "autonym":"Marwari",
+    "language_name":"Haryanvi",
+    "autonym":"हरियाणवी",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -888,10 +888,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bgc",
+    "bcp_47":"mwr",
     "speakers":15913080,
-    "language_name":"Haryanvi",
-    "autonym":"हरियाणवी",
+    "language_name":"Marwari",
+    "autonym":"Marwari",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -1073,11 +1073,11 @@
     "language_name":"Akan",
     "autonym":"Akan",
     "family":"Atlantic-Congo",
-    "flores_path":"twi_Latn",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":0.2,
     "commonvoice_locale":"tw",
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"qu",
@@ -1195,7 +1195,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"lua_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":0.7,
+    "commonvoice_hours":0.8,
     "commonvoice_locale":"lua",
     "in_benchmark":true
   },
@@ -1303,7 +1303,7 @@
     "family":"Afro-Asiatic",
     "flores_path":"heb_Hebr",
     "fleurs_tag":"he_il",
-    "commonvoice_hours":1.2,
+    "commonvoice_hours":1.3,
     "commonvoice_locale":"he",
     "in_benchmark":true
   },
@@ -1375,7 +1375,7 @@
     "family":"Turkic",
     "flores_path":"uig_Arab",
     "fleurs_tag":null,
-    "commonvoice_hours":371.0,
+    "commonvoice_hours":372.0,
     "commonvoice_locale":"ug",
     "in_benchmark":true
   },
@@ -1555,7 +1555,7 @@
     "family":"Indo-European",
     "flores_path":"slk_Latn",
     "fleurs_tag":"sk_sk",
-    "commonvoice_hours":48.0,
+    "commonvoice_hours":49.0,
     "commonvoice_locale":"sk",
     "in_benchmark":true
   },
@@ -1955,18 +1955,6 @@
     "commonvoice_locale":"gom",
     "in_benchmark":true
   },
-  {
-    "bcp_47":"kln",
-    "speakers":4068120,
-    "language_name":"Kalenjin",
-    "autonym":"Kalenjin",
-    "family":"Nilotic",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":43.0,
-    "commonvoice_locale":"kln",
-    "in_benchmark":false
-  },
   {
     "bcp_47":"kam",
     "speakers":4068120,
@@ -1979,6 +1967,18 @@
     "commonvoice_locale":"kam",
     "in_benchmark":true
   },
+  {
+    "bcp_47":"kln",
+    "speakers":4068120,
+    "language_name":"Kalenjin",
+    "autonym":"Kalenjin",
+    "family":"Nilotic",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":43.0,
+    "commonvoice_locale":"kln",
+    "in_benchmark":false
+  },
   {
     "bcp_47":"bjn",
     "speakers":4010288,
@@ -2009,11 +2009,11 @@
     "language_name":"Makhuwa",
     "autonym":"Emakhuwa",
     "family":"Atlantic-Congo",
-    "flores_path":"vmw_Latn",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":0.0,
     "commonvoice_locale":"vmw",
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"glk",
@@ -2124,10 +2124,10 @@
     "in_benchmark":true
   },
   {
-    "bcp_47":"gbm",
+    "bcp_47":"lmn",
     "speakers":3580443,
-    "language_name":"Garhwali",
-    "autonym":"Garhwali",
+    "language_name":"Lambadi",
+    "autonym":"Lambadi",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2136,10 +2136,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"lmn",
+    "bcp_47":"gbm",
     "speakers":3580443,
-    "language_name":"Lambadi",
-    "autonym":"Lambadi",
+    "language_name":"Garhwali",
+    "autonym":"Garhwali",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2335,7 +2335,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":2.5,
+    "commonvoice_hours":4.5,
     "commonvoice_locale":"bci",
     "in_benchmark":false
   },
@@ -2352,27 +2352,27 @@
     "in_benchmark":true
   },
   {
-    "bcp_47":"efi",
+    "bcp_47":"ibb",
     "speakers":2996392,
-    "language_name":"Efik",
-    "autonym":"Efik",
+    "language_name":"Ibibio",
+    "autonym":"Ibibio",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":5.5,
+    "commonvoice_locale":"ibb",
     "in_benchmark":false
   },
   {
-    "bcp_47":"ibb",
+    "bcp_47":"efi",
     "speakers":2996392,
-    "language_name":"Ibibio",
-    "autonym":"Ibibio",
+    "language_name":"Efik",
+    "autonym":"Efik",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":5.3,
-    "commonvoice_locale":"ibb",
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
     "in_benchmark":false
   },
   {
@@ -2544,11 +2544,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"wbq",
+    "bcp_47":"sck",
     "speakers":2386962,
-    "language_name":"Waddar",
-    "autonym":"Waddar",
-    "family":"Dravidian",
+    "language_name":"Sadri",
+    "autonym":"Sadri",
+    "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -2556,11 +2556,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"sck",
+    "bcp_47":"wbq",
     "speakers":2386962,
-    "language_name":"Sadri",
-    "autonym":"Sadri",
-    "family":"Indo-European",
+    "language_name":"Waddar",
+    "autonym":"Waddar",
+    "family":"Dravidian",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -2724,10 +2724,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"khn",
+    "bcp_47":"wbr",
     "speakers":1989135,
-    "language_name":"Khandesi",
-    "autonym":"Khandesi",
+    "language_name":"Wagdi",
+    "autonym":"Wagdi",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -2748,10 +2748,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"wbr",
+    "bcp_47":"khn",
     "speakers":1989135,
-    "language_name":"Wagdi",
-    "autonym":"Wagdi",
+    "language_name":"Khandesi",
+    "autonym":"Khandesi",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -3283,7 +3283,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":10.0,
+    "commonvoice_hours":11.0,
     "commonvoice_locale":"bum",
     "in_benchmark":false
   },
@@ -3535,7 +3535,7 @@
     "family":null,
     "flores_path":"eus_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":358.0,
+    "commonvoice_hours":365.0,
     "commonvoice_locale":"eu",
     "in_benchmark":true
   },
@@ -3559,7 +3559,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":56.0,
+    "commonvoice_hours":58.0,
     "commonvoice_locale":"kbd",
     "in_benchmark":false
   },
@@ -3679,15 +3679,15 @@
     "family":"Indo-European",
     "flores_path":"ydd_Hebr",
     "fleurs_tag":null,
-    "commonvoice_hours":0.5,
+    "commonvoice_hours":0.6,
     "commonvoice_locale":"yi",
     "in_benchmark":true
   },
   {
-    "bcp_47":"ksb",
+    "bcp_47":"bez",
     "speakers":995398,
-    "language_name":"Shambala",
-    "autonym":"Kishambaa",
+    "language_name":"Bena",
+    "autonym":"Hibena",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
@@ -3696,10 +3696,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bez",
+    "bcp_47":"ksb",
     "speakers":995398,
-    "language_name":"Bena",
-    "autonym":"Hibena",
+    "language_name":"Shambala",
+    "autonym":"Kishambaa",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
@@ -4512,27 +4512,27 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"nhw",
+    "bcp_47":"nhe",
     "speakers":501735,
-    "language_name":"Western Huasteca Nahuatl",
-    "autonym":"Western Huasteca Nahuatl",
+    "language_name":"Eastern Huasteca Nahuatl",
+    "autonym":"Eastern Huasteca Nahuatl",
     "family":"Uto-Aztecan",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"nhe",
     "in_benchmark":false
   },
   {
-    "bcp_47":"nhe",
+    "bcp_47":"nhw",
     "speakers":501735,
-    "language_name":"Eastern Huasteca Nahuatl",
-    "autonym":"Eastern Huasteca Nahuatl",
+    "language_name":"Western Huasteca Nahuatl",
+    "autonym":"Western Huasteca Nahuatl",
     "family":"Uto-Aztecan",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":0.0,
-    "commonvoice_locale":"nhe",
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
     "in_benchmark":false
   },
   {
@@ -4651,7 +4651,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":15.0,
+    "commonvoice_hours":16.0,
     "commonvoice_locale":"ady",
     "in_benchmark":false
   },
@@ -4675,7 +4675,7 @@
     "family":"Kartvelian",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":11.0,
+    "commonvoice_hours":12.0,
     "commonvoice_locale":"xmf",
     "in_benchmark":false
   },
@@ -4685,11 +4685,11 @@
     "language_name":"Erzya",
     "autonym":"Эрзянь Кель",
     "family":"Uralic",
-    "flores_path":"myv_Cyrl",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":3.8,
     "commonvoice_locale":"myv",
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"dav",
@@ -4715,18 +4715,6 @@
     "commonvoice_locale":null,
     "in_benchmark":false
   },
-  {
-    "bcp_47":"jmc",
-    "speakers":433291,
-    "language_name":"Machame",
-    "autonym":"Kimachame",
-    "family":"Atlantic-Congo",
-    "flores_path":null,
-    "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
-    "in_benchmark":false
-  },
   {
     "bcp_47":"vun",
     "speakers":433291,
@@ -4751,6 +4739,18 @@
     "commonvoice_locale":"rof",
     "in_benchmark":false
   },
+  {
+    "bcp_47":"jmc",
+    "speakers":433291,
+    "language_name":"Machame",
+    "autonym":"Kimachame",
+    "family":"Atlantic-Congo",
+    "flores_path":null,
+    "fleurs_tag":null,
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
+    "in_benchmark":false
+  },
   {
     "bcp_47":"kjg",
     "speakers":431949,
@@ -5009,11 +5009,11 @@
     "language_name":"Dargwa",
     "autonym":"Dargwa",
     "family":"Nakh-Daghestanian",
-    "flores_path":"dar_Cyrl",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":0.0,
     "commonvoice_locale":"dar",
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"lif",
@@ -5124,27 +5124,27 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bas",
+    "bcp_47":"bax",
     "speakers":332940,
-    "language_name":"Basaa",
-    "autonym":"Ɓàsàa",
+    "language_name":"Bamun",
+    "autonym":"Bamun",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":12.0,
-    "commonvoice_locale":"bas",
+    "commonvoice_hours":11.0,
+    "commonvoice_locale":"bax",
     "in_benchmark":false
   },
   {
-    "bcp_47":"bax",
+    "bcp_47":"bas",
     "speakers":332940,
-    "language_name":"Bamun",
-    "autonym":"Bamun",
+    "language_name":"Basaa",
+    "autonym":"Ɓàsàa",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":11.0,
-    "commonvoice_locale":"bax",
+    "commonvoice_hours":12.0,
+    "commonvoice_locale":"bas",
     "in_benchmark":false
   },
   {
@@ -5232,11 +5232,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bfq",
+    "bcp_47":"njo",
     "speakers":305001,
-    "language_name":"Badaga",
-    "autonym":"Badaga",
-    "family":"Dravidian",
+    "language_name":"Ao Naga",
+    "autonym":"Ao Naga",
+    "family":"Sino-Tibetan",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -5244,11 +5244,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"njo",
+    "bcp_47":"bfq",
     "speakers":305001,
-    "language_name":"Ao Naga",
-    "autonym":"Ao Naga",
-    "family":"Sino-Tibetan",
+    "language_name":"Badaga",
+    "autonym":"Badaga",
+    "family":"Dravidian",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -5388,10 +5388,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"tdd",
+    "bcp_47":"khb",
     "speakers":264864,
-    "language_name":"Tai Nüa",
-    "autonym":"Tai Nüa",
+    "language_name":"Lü",
+    "autonym":"Lü",
     "family":"Tai-Kadai",
     "flores_path":null,
     "fleurs_tag":null,
@@ -5400,10 +5400,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"khb",
+    "bcp_47":"tdd",
     "speakers":264864,
-    "language_name":"Lü",
-    "autonym":"Lü",
+    "language_name":"Tai Nüa",
+    "autonym":"Tai Nüa",
     "family":"Tai-Kadai",
     "flores_path":null,
     "fleurs_tag":null,
@@ -5508,10 +5508,10 @@
     "in_benchmark":true
   },
   {
-    "bcp_47":"sxn",
+    "bcp_47":"mdr",
     "speakers":245664,
-    "language_name":"Sangir",
-    "autonym":"Sangir",
+    "language_name":"Mandar",
+    "autonym":"Mandar",
     "family":"Austronesian",
     "flores_path":null,
     "fleurs_tag":null,
@@ -5520,10 +5520,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"mdr",
+    "bcp_47":"sxn",
     "speakers":245664,
-    "language_name":"Mandar",
-    "autonym":"Mandar",
+    "language_name":"Sangir",
+    "autonym":"Sangir",
     "family":"Austronesian",
     "flores_path":null,
     "fleurs_tag":null,
@@ -5753,11 +5753,11 @@
     "language_name":"Tuvinian",
     "autonym":"Tuvinian",
     "family":"Turkic",
-    "flores_path":"tyv_Cyrl",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":0.0,
     "commonvoice_locale":"tyv",
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"dtp",
@@ -5904,10 +5904,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bss",
+    "bcp_47":"kkj",
     "speakers":149823,
-    "language_name":"Akoose",
-    "autonym":"Akoose",
+    "language_name":"Kako",
+    "autonym":"Kakɔ",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
@@ -5916,10 +5916,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"kkj",
+    "bcp_47":"bss",
     "speakers":149823,
-    "language_name":"Kako",
-    "autonym":"Kakɔ",
+    "language_name":"Akoose",
+    "autonym":"Akoose",
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
@@ -6223,7 +6223,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":68.0,
+    "commonvoice_hours":67.0,
     "commonvoice_locale":"ab",
     "in_benchmark":false
   },
@@ -6821,11 +6821,11 @@
     "language_name":"Aragonese",
     "autonym":"Aragonés",
     "family":"Indo-European",
-    "flores_path":"arg_Latn",
+    "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":17.0,
     "commonvoice_locale":"an",
-    "in_benchmark":true
+    "in_benchmark":false
   },
   {
     "bcp_47":"chr",
@@ -7272,11 +7272,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"bku",
+    "bcp_47":"twq",
     "speakers":7970,
-    "language_name":"Buhid",
-    "autonym":"Buhid",
-    "family":"Austronesian",
+    "language_name":"Tasawaq",
+    "autonym":"Tasawaq Senni",
+    "family":"Songhay",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7284,11 +7284,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"twq",
+    "bcp_47":"bku",
     "speakers":7970,
-    "language_name":"Tasawaq",
-    "autonym":"Tasawaq Senni",
-    "family":"Songhay",
+    "language_name":"Buhid",
+    "autonym":"Buhid",
+    "family":"Austronesian",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7567,7 +7567,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":1.7,
+    "commonvoice_hours":2.4,
     "commonvoice_locale":"yav",
     "in_benchmark":false
   },
@@ -7836,11 +7836,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"crl",
+    "bcp_47":"kwk",
     "speakers":377,
-    "language_name":"Northern East Cree",
-    "autonym":"Northern East Cree",
-    "family":"Algic",
+    "language_name":"Kwakʼwala",
+    "autonym":"KwakʼWala",
+    "family":"Wakashan",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7848,11 +7848,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"kwk",
+    "bcp_47":"crl",
     "speakers":377,
-    "language_name":"Kwakʼwala",
-    "autonym":"KwakʼWala",
-    "family":"Wakashan",
+    "language_name":"Northern East Cree",
+    "autonym":"Northern East Cree",
+    "family":"Algic",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7968,11 +7968,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"sgs",
+    "bcp_47":"lzh",
     "speakers":0,
-    "language_name":"Samogitian",
-    "autonym":"Samogitian",
-    "family":"Indo-European",
+    "language_name":"Literary Chinese",
+    "autonym":"Literary Chinese",
+    "family":"Sino-Tibetan",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7980,11 +7980,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"rgn",
+    "bcp_47":"io",
     "speakers":0,
-    "language_name":"Romagnol",
-    "autonym":"Romagnol",
-    "family":"Indo-European",
+    "language_name":"Ido",
+    "autonym":"Ido",
+    "family":"Artificial Language",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -7992,22 +7992,22 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"ann",
+    "bcp_47":"jbo",
     "speakers":0,
-    "language_name":"Obolo",
-    "autonym":"Obolo",
-    "family":"Atlantic-Congo",
+    "language_name":"Lojban",
+    "autonym":"La .Lojban.",
+    "family":"Artificial Language",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"jbo",
     "in_benchmark":false
   },
   {
-    "bcp_47":"pfl",
+    "bcp_47":"jut",
     "speakers":0,
-    "language_name":"Palatine German",
-    "autonym":"Palatine German",
+    "language_name":"Jutish",
+    "autonym":"Jutish",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -8016,23 +8016,23 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"osa",
+    "bcp_47":"vot",
     "speakers":0,
-    "language_name":"Osage",
-    "autonym":"𐓏𐓘𐓻𐓘𐓻𐓟",
-    "family":"Siouan",
+    "language_name":"Votic",
+    "autonym":"Votic",
+    "family":"Uralic",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.1,
+    "commonvoice_locale":"vot",
     "in_benchmark":false
   },
   {
-    "bcp_47":"lzh",
+    "bcp_47":"gez",
     "speakers":0,
-    "language_name":"Literary Chinese",
-    "autonym":"Literary Chinese",
-    "family":"Sino-Tibetan",
+    "language_name":"Geez",
+    "autonym":"Geez",
+    "family":"Afro-Asiatic",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -8040,23 +8040,23 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"jbo",
+    "bcp_47":"osa",
     "speakers":0,
-    "language_name":"Lojban",
-    "autonym":"La .Lojban.",
-    "family":"Artificial Language",
+    "language_name":"Osage",
+    "autonym":"𐓏𐓘𐓻𐓘𐓻𐓟",
+    "family":"Siouan",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":0.0,
-    "commonvoice_locale":"jbo",
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
     "in_benchmark":false
   },
   {
-    "bcp_47":"io",
+    "bcp_47":"rgn",
     "speakers":0,
-    "language_name":"Ido",
-    "autonym":"Ido",
-    "family":"Artificial Language",
+    "language_name":"Romagnol",
+    "autonym":"Romagnol",
+    "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -8064,10 +8064,10 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"jut",
+    "bcp_47":"cu",
     "speakers":0,
-    "language_name":"Jutish",
-    "autonym":"Jutish",
+    "language_name":"Church Slavic",
+    "autonym":"Church Slavic",
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
@@ -8076,11 +8076,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"gez",
+    "bcp_47":"sgs",
     "speakers":0,
-    "language_name":"Geez",
-    "autonym":"Geez",
-    "family":"Afro-Asiatic",
+    "language_name":"Samogitian",
+    "autonym":"Samogitian",
+    "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -8088,11 +8088,11 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"cu",
+    "bcp_47":"ann",
     "speakers":0,
-    "language_name":"Church Slavic",
-    "autonym":"Church Slavic",
-    "family":"Indo-European",
+    "language_name":"Obolo",
+    "autonym":"Obolo",
+    "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,
@@ -8112,23 +8112,23 @@
     "in_benchmark":false
   },
   {
-    "bcp_47":"vot",
+    "bcp_47":"cad",
     "speakers":0,
-    "language_name":"Votic",
-    "autonym":"Votic",
-    "family":"Uralic",
+    "language_name":"Caddo",
+    "autonym":"Caddo",
+    "family":"Caddoan",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":0.1,
-    "commonvoice_locale":"vot",
+    "commonvoice_hours":null,
+    "commonvoice_locale":null,
     "in_benchmark":false
   },
   {
-    "bcp_47":"cad",
+    "bcp_47":"pfl",
     "speakers":0,
-    "language_name":"Caddo",
-    "autonym":"Caddo",
-    "family":"Caddoan",
+    "language_name":"Palatine German",
+    "autonym":"Palatine German",
+    "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
     "commonvoice_hours":null,

models.json

@@ -1,157 +1,102 @@
 [
   {
-    "id":"meta-llama\/llama-4-maverick",
-    "name":"Llama 4 Maverick (free)",
-    "provider_name":"Meta",
-    "cost":0.0,
-    "hf_id":"meta-llama\/Llama-4-Maverick-17B-128E-Instruct",
-    "size":401583781376.0,
-    "type":"Open",
-    "license":"Other",
-    "creation_date":1743465600000
+    "id":"amazon\/nova-micro-v1",
+    "name":"Nova Micro 1.0",
+    "provider_name":"Amazon",
+    "cost":0.14,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1733356800000
   },
   {
-    "id":"meta-llama\/llama-3.3-70b-instruct",
-    "name":"Llama 3.3 70B Instruct (free)",
-    "provider_name":"Meta",
+    "id":"deepseek\/deepseek-chat",
+    "name":"DeepSeek V3 (free)",
+    "provider_name":"DeepSeek",
     "cost":0.0,
-    "hf_id":"meta-llama\/Llama-3.3-70B-Instruct",
-    "size":70553706496.0,
+    "hf_id":"deepseek-ai\/DeepSeek-V3",
+    "size":684531386000.0,
     "type":"Open",
-    "license":"Llama3.3",
-    "creation_date":1732579200000
+    "license":"",
+    "creation_date":1735084800000
   },
   {
-    "id":"meta-llama\/llama-3.1-70b-instruct",
-    "name":"Llama 3.1 70B Instruct",
-    "provider_name":"Meta",
-    "cost":0.28,
-    "hf_id":"meta-llama\/Llama-3.1-70B-Instruct",
-    "size":70553706496.0,
+    "id":"deepseek\/deepseek-chat-v3-0324",
+    "name":"DeepSeek V3 0324 (free)",
+    "provider_name":"DeepSeek",
+    "cost":0.0,
+    "hf_id":"deepseek-ai\/DeepSeek-V3-0324",
+    "size":684531386000.0,
     "type":"Open",
-    "license":"Llama3.1",
-    "creation_date":1721088000000
+    "license":"Mit",
+    "creation_date":1742774400000
   },
   {
-    "id":"meta-llama\/llama-3-70b-instruct",
-    "name":"Llama 3 70B Instruct",
-    "provider_name":"Meta",
-    "cost":0.4,
-    "hf_id":"meta-llama\/Meta-Llama-3-70B-Instruct",
-    "size":70553706496.0,
+    "id":"deepseek\/deepseek-r1",
+    "name":"R1 (free)",
+    "provider_name":"DeepSeek",
+    "cost":0.0,
+    "hf_id":"deepseek-ai\/DeepSeek-R1",
+    "size":684531386000.0,
     "type":"Open",
-    "license":"Llama3",
-    "creation_date":1713312000000
-  },
-  {
-    "id":"openai\/gpt-4.1-mini",
-    "name":"GPT-4.1 Mini",
-    "provider_name":"OpenAI",
-    "cost":1.6,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1744588800000
+    "license":"Mit",
+    "creation_date":1737331200000
   },
   {
-    "id":"openai\/gpt-4.1-nano",
-    "name":"GPT-4.1 Nano",
-    "provider_name":"OpenAI",
+    "id":"google\/gemini-2.0-flash-001",
+    "name":"Gemini 2.0 Flash",
+    "provider_name":"Google",
     "cost":0.4,
     "hf_id":null,
     "size":null,
     "type":"Commercial",
     "license":null,
-    "creation_date":1744588800000
-  },
-  {
-    "id":"openai\/gpt-4o-mini",
-    "name":"GPT-4o-mini",
-    "provider_name":"OpenAI",
-    "cost":0.6,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1721260800000
-  },
-  {
-    "id":"openai\/gpt-3.5-turbo-0613",
-    "name":"GPT-3.5 Turbo (older v0613)",
-    "provider_name":"OpenAI",
-    "cost":2.0,
-    "hf_id":null,
-    "size":null,
-    "type":"Commercial",
-    "license":null,
-    "creation_date":1706140800000
+    "creation_date":1738713600000
   },
   {
-    "id":"openai\/gpt-3.5-turbo",
-    "name":"GPT-3.5 Turbo",
-    "provider_name":"OpenAI",
-    "cost":1.5,
+    "id":"google\/gemini-2.0-flash-lite-001",
+    "name":"Gemini 2.0 Flash Lite",
+    "provider_name":"Google",
+    "cost":0.3,
     "hf_id":null,
     "size":null,
     "type":"Commercial",
     "license":null,
-    "creation_date":1685232000000
-  },
-  {
-    "id":"mistralai\/mistral-small-3.1-24b-instruct",
-    "name":"Mistral Small 3.1 24B (free)",
-    "provider_name":"Mistral",
-    "cost":0.0,
-    "hf_id":"mistralai\/Mistral-Small-3.1-24B-Instruct-2503",
-    "size":24011361280.0,
-    "type":"Open",
-    "license":"Apache 2.0",
-    "creation_date":1741651200000
+    "creation_date":1740441600000
   },
   {
-    "id":"mistralai\/mistral-saba",
-    "name":"Saba",
-    "provider_name":"Mistral",
+    "id":"google\/gemini-2.5-flash-preview",
+    "name":"Gemini 2.5 Flash Preview",
+    "provider_name":"Google",
     "cost":0.6,
     "hf_id":null,
     "size":null,
     "type":"Commercial",
     "license":null,
-    "creation_date":1739750400000
-  },
-  {
-    "id":"mistralai\/mistral-nemo",
-    "name":"Mistral Nemo (free)",
-    "provider_name":"Mistral",
-    "cost":0.0,
-    "hf_id":"mistralai\/Mistral-Nemo-Instruct-2407",
-    "size":12247782400.0,
-    "type":"Open",
-    "license":"Apache 2.0",
-    "creation_date":1721174400000
+    "creation_date":1744848000000
   },
   {
-    "id":"google\/gemini-2.5-flash-preview",
-    "name":"Gemini 2.5 Flash Preview",
+    "id":"google\/gemini-flash-1.5",
+    "name":"Gemini 1.5 Flash ",
     "provider_name":"Google",
-    "cost":0.6,
+    "cost":0.3,
     "hf_id":null,
     "size":null,
     "type":"Commercial",
     "license":null,
-    "creation_date":1744848000000
+    "creation_date":1715644800000
   },
   {
-    "id":"google\/gemini-2.0-flash-lite-001",
-    "name":"Gemini 2.0 Flash Lite",
+    "id":"google\/gemini-flash-1.5-8b",
+    "name":"Gemini 1.5 Flash 8B",
     "provider_name":"Google",
-    "cost":0.3,
+    "cost":0.15,
     "hf_id":null,
     "size":null,
     "type":"Commercial",
     "license":null,
-    "creation_date":1740441600000
+    "creation_date":1727913600000
   },
   {
     "id":"google\/gemma-3-27b-it",
@@ -165,26 +110,48 @@
     "creation_date":1740787200000
   },
   {
-    "id":"deepseek\/deepseek-chat-v3-0324",
-    "name":"DeepSeek V3 0324 (free)",
-    "provider_name":"DeepSeek",
+    "id":"meta-llama\/llama-3-70b-instruct",
+    "name":"Llama 3 70B Instruct",
+    "provider_name":"Meta",
+    "cost":0.4,
+    "hf_id":"meta-llama\/Meta-Llama-3-70B-Instruct",
+    "size":70553706496.0,
+    "type":"Open",
+    "license":"Llama3",
+    "creation_date":1713312000000
+  },
+  {
+    "id":"meta-llama\/llama-3.1-70b-instruct",
+    "name":"Llama 3.1 70B Instruct",
+    "provider_name":"Meta",
+    "cost":0.28,
+    "hf_id":"meta-llama\/Llama-3.1-70B-Instruct",
+    "size":70553706496.0,
+    "type":"Open",
+    "license":"Llama3.1",
+    "creation_date":1721088000000
+  },
+  {
+    "id":"meta-llama\/llama-3.3-70b-instruct",
+    "name":"Llama 3.3 70B Instruct (free)",
+    "provider_name":"Meta",
     "cost":0.0,
-    "hf_id":"deepseek-ai\/DeepSeek-V3-0324",
-    "size":684531386000.0,
+    "hf_id":"meta-llama\/Llama-3.3-70B-Instruct",
+    "size":70553706496.0,
     "type":"Open",
-    "license":"Mit",
-    "creation_date":1742774400000
+    "license":"Llama3.3",
+    "creation_date":1732579200000
   },
   {
-    "id":"deepseek\/deepseek-chat",
-    "name":"DeepSeek V3 (free)",
-    "provider_name":"DeepSeek",
+    "id":"meta-llama\/llama-4-maverick",
+    "name":"Llama 4 Maverick (free)",
+    "provider_name":"Meta",
     "cost":0.0,
-    "hf_id":"deepseek-ai\/DeepSeek-V3",
-    "size":684531386000.0,
+    "hf_id":"meta-llama\/Llama-4-Maverick-17B-128E-Instruct",
+    "size":401583781376.0,
     "type":"Open",
-    "license":"",
-    "creation_date":1735084800000
+    "license":"Other",
+    "creation_date":1743465600000
   },
   {
     "id":"microsoft\/phi-4",
@@ -209,14 +176,91 @@
     "creation_date":1740355200000
   },
   {
-    "id":"amazon\/nova-micro-v1",
-    "name":"Nova Micro 1.0",
-    "provider_name":"Amazon",
-    "cost":0.14,
+    "id":"mistralai\/mistral-nemo",
+    "name":"Mistral Nemo (free)",
+    "provider_name":"Mistral",
+    "cost":0.0,
+    "hf_id":"mistralai\/Mistral-Nemo-Instruct-2407",
+    "size":12247782400.0,
+    "type":"Open",
+    "license":"Apache 2.0",
+    "creation_date":1721174400000
+  },
+  {
+    "id":"mistralai\/mistral-saba",
+    "name":"Saba",
+    "provider_name":"Mistral",
+    "cost":0.6,
     "hf_id":null,
     "size":null,
     "type":"Commercial",
     "license":null,
-    "creation_date":1733356800000
+    "creation_date":1739750400000
+  },
+  {
+    "id":"mistralai\/mistral-small-3.1-24b-instruct",
+    "name":"Mistral Small 3.1 24B (free)",
+    "provider_name":"Mistral",
+    "cost":0.0,
+    "hf_id":"mistralai\/Mistral-Small-3.1-24B-Instruct-2503",
+    "size":24011361280.0,
+    "type":"Open",
+    "license":"Apache 2.0",
+    "creation_date":1741651200000
+  },
+  {
+    "id":"openai\/gpt-3.5-turbo",
+    "name":"GPT-3.5 Turbo",
+    "provider_name":"OpenAI",
+    "cost":1.5,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1685232000000
+  },
+  {
+    "id":"openai\/gpt-3.5-turbo-0613",
+    "name":"GPT-3.5 Turbo (older v0613)",
+    "provider_name":"OpenAI",
+    "cost":2.0,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1706140800000
+  },
+  {
+    "id":"openai\/gpt-4.1-mini",
+    "name":"GPT-4.1 Mini",
+    "provider_name":"OpenAI",
+    "cost":1.6,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1744588800000
+  },
+  {
+    "id":"openai\/gpt-4.1-nano",
+    "name":"GPT-4.1 Nano",
+    "provider_name":"OpenAI",
+    "cost":0.4,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1744588800000
+  },
+  {
+    "id":"openai\/gpt-4o-mini",
+    "name":"GPT-4o-mini",
+    "provider_name":"OpenAI",
+    "cost":0.6,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1721260800000
   }
 ]