Upload from GitHub Actions: Merge pull request #7 from datenlabor-bmz/jn-dev

.gitattributes

@@ -0,0 +1 @@
+evals/data_flow_architecture.png filter=lfs diff=lfs merge=lfs -text

evals/models.py

@@ -93,28 +93,91 @@ def get_model(permaslug):
 
 @cache
 def get_historical_popular_models(date: date):
-    raw = get("https://openrouter.ai/rankings").text
-    data = re.search(r'{\\"data\\":(.*),\\"isPercentage\\"', raw).group(1)
-    data = json.loads(data.replace("\\", ""))
-    counts = defaultdict(int)
-    for day in data:
-        for model, count in day["ys"].items():
-            if model.startswith("openrouter") or model == "Others":
-                continue
-            counts[model.split(":")[0]] += count
-    counts = sorted(counts.items(), key=lambda x: x[1], reverse=True)
-    models = [get_model(model) for model, _ in counts]
-    return [m for m in models if m]
-
-
-@cache
+    try:
+        raw = get("https://openrouter.ai/rankings").text
+        
+        # Extract model data from rankingData using regex
+        import re
+        import json
+        
+        # Find all count and model_permaslug pairs in the data
+        # Format: "count":number,"model_permaslug":"model/name"
+        pattern = r'\\\"count\\\":([\d.]+).*?\\\"model_permaslug\\\":\\\"([^\\\"]+)\\\"'
+        matches = re.findall(pattern, raw)
+        
+        if matches:
+            # Aggregate model counts
+            model_counts = {}
+            for count_str, model_slug in matches:
+                count = float(count_str)
+                if not model_slug.startswith('openrouter') and model_slug != 'Others':
+                    # Remove variant suffixes for aggregation
+                    base_model = model_slug.split(':')[0]
+                    model_counts[base_model] = model_counts.get(base_model, 0) + count
+            
+            # Sort by popularity and return top models
+            sorted_models = sorted(model_counts.items(), key=lambda x: x[1], reverse=True)
+            result = []
+            for model_slug, count in sorted_models[:20]:  # Top 20
+                result.append({"slug": model_slug, "count": int(count)})
+            
+            print(f"✅ Historical OpenRouter models: {len(result)} models fetched")
+            if result:
+                print(f"   Top 5: {[m['slug'] for m in result[:5]]}")
+                print(f"   Sample counts: {[m['count'] for m in result[:3]]}")
+            return result
+        else:
+            print("⚠️ Could not find model ranking data in OpenRouter response")
+            return []
+        
+    except Exception as e:
+        print(f"⚠️ Error fetching OpenRouter historical rankings: {e}")
+        print("🔄 Falling back to static model list")
+        return []
+
+
+@cache  
 def get_current_popular_models(date: date):
-    raw = get("https://openrouter.ai/rankings?view=day").text.replace("\\", "")
-    data = re.search(r'"rankingData":(.*),"rankingType":"day"', raw).group(1)
-    data = json.loads(data)
-    data = sorted(data, key=lambda x: x["total_prompt_tokens"], reverse=True)
-    models = [get_model(model["model_permaslug"]) for model in data]
-    return [m for m in models if m]
+    try:
+        raw = get("https://openrouter.ai/rankings?view=day").text
+        
+        # Extract model data from daily rankings
+        import re
+        import json
+        
+        # Find all count and model_permaslug pairs in the daily data
+        pattern = r'\\\"count\\\":([\d.]+).*?\\\"model_permaslug\\\":\\\"([^\\\"]+)\\\"'
+        matches = re.findall(pattern, raw)
+        
+        if matches:
+            # Aggregate model counts
+            model_counts = {}
+            for count_str, model_slug in matches:
+                count = float(count_str)
+                if not model_slug.startswith('openrouter') and model_slug != 'Others':
+                    # Remove variant suffixes for aggregation
+                    base_model = model_slug.split(':')[0]
+                    model_counts[base_model] = model_counts.get(base_model, 0) + count
+            
+            # Sort by popularity and return top models
+            sorted_models = sorted(model_counts.items(), key=lambda x: x[1], reverse=True)
+            result = []
+            for model_slug, count in sorted_models[:10]:  # Top 10
+                result.append({"slug": model_slug, "count": int(count)})
+            
+            print(f"✅ Current OpenRouter models: {len(result)} models fetched")
+            if result:
+                print(f"   Top 5: {[m['slug'] for m in result[:5]]}")
+                print(f"   Sample counts: {[m['count'] for m in result[:3]]}")
+            return result
+        else:
+            print("⚠️ Could not find daily ranking data in OpenRouter response")
+            return []
+        
+    except Exception as e:
+        print(f"⚠️ Error fetching OpenRouter current rankings: {e}")
+        print("🔄 Falling back to static model list")
+        return []
 
 
 def get_translation_models():
@@ -249,8 +312,14 @@ def get_hf_metadata(row):
 
 
 def get_cost(row):
-    cost = float(row["endpoint"]["pricing"]["completion"])
-    return round(cost * 1_000_000, 2)
+    """
+    row: a row from the OpenRouter models dataframe
+    """
+    try:
+        cost = float(row["endpoint"]["pricing"]["completion"])
+        return round(cost * 1_000_000, 2)
+    except (TypeError, KeyError):
+        return None
 
 
 @cache

languages.json

@@ -7,7 +7,7 @@
     "family":"Indo-European",
     "flores_path":"eng_Latn",
     "fleurs_tag":"en_us",
-    "commonvoice_hours":2674.0,
+    "commonvoice_hours":2678.0,
     "commonvoice_locale":"en",
     "in_benchmark":true
   },
@@ -79,7 +79,7 @@
     "family":"Indo-European",
     "flores_path":"fra_Latn",
     "fleurs_tag":"fr_fr",
-    "commonvoice_hours":1065.0,
+    "commonvoice_hours":1067.0,
     "commonvoice_locale":"fr",
     "in_benchmark":true
   },
@@ -103,7 +103,7 @@
     "family":"Indo-European",
     "flores_path":"por_Latn",
     "fleurs_tag":"pt_br",
-    "commonvoice_hours":180.0,
+    "commonvoice_hours":181.0,
     "commonvoice_locale":"pt",
     "in_benchmark":true
   },
@@ -115,7 +115,7 @@
     "family":"Indo-European",
     "flores_path":"pan_Guru",
     "fleurs_tag":"pa_in",
-    "commonvoice_hours":2.3,
+    "commonvoice_hours":2.5,
     "commonvoice_locale":"pa-IN",
     "in_benchmark":true
   },
@@ -163,7 +163,7 @@
     "family":"Indo-European",
     "flores_path":"deu_Latn",
     "fleurs_tag":"de_de",
-    "commonvoice_hours":1369.0,
+    "commonvoice_hours":1370.0,
     "commonvoice_locale":"de",
     "in_benchmark":true
   },
@@ -379,7 +379,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":"ps_af",
-    "commonvoice_hours":81.0,
+    "commonvoice_hours":82.0,
     "commonvoice_locale":"ps",
     "in_benchmark":false
   },
@@ -619,7 +619,7 @@
     "family":"Indo-European",
     "flores_path":"nld_Latn",
     "fleurs_tag":"nl_nl",
-    "commonvoice_hours":120.0,
+    "commonvoice_hours":122.0,
     "commonvoice_locale":"nl",
     "in_benchmark":true
   },
@@ -655,7 +655,7 @@
     "family":"Atlantic-Congo",
     "flores_path":"yor_Latn",
     "fleurs_tag":"yo_ng",
-    "commonvoice_hours":6.3,
+    "commonvoice_hours":6.4,
     "commonvoice_locale":"yo",
     "in_benchmark":true
   },
@@ -1099,7 +1099,7 @@
     "family":"Indo-European",
     "flores_path":"ckb_Arab",
     "fleurs_tag":"ckb_iq",
-    "commonvoice_hours":135.0,
+    "commonvoice_hours":136.0,
     "commonvoice_locale":"ckb",
     "in_benchmark":true
   },
@@ -1183,7 +1183,7 @@
     "family":"Indo-European",
     "flores_path":"bel_Cyrl",
     "fleurs_tag":"be_by",
-    "commonvoice_hours":1810.0,
+    "commonvoice_hours":1811.0,
     "commonvoice_locale":"be",
     "in_benchmark":true
   },
@@ -1243,7 +1243,7 @@
     "family":"Indo-European",
     "flores_path":"afr_Latn",
     "fleurs_tag":"af_za",
-    "commonvoice_hours":0.5,
+    "commonvoice_hours":0.6,
     "commonvoice_locale":"af",
     "in_benchmark":true
   },
@@ -1291,7 +1291,7 @@
     "family":"Indo-European",
     "flores_path":"cat_Latn",
     "fleurs_tag":"ca_es",
-    "commonvoice_hours":2863.0,
+    "commonvoice_hours":2874.0,
     "commonvoice_locale":"ca",
     "in_benchmark":true
   },
@@ -1303,7 +1303,7 @@
     "family":"Afro-Asiatic",
     "flores_path":"heb_Hebr",
     "fleurs_tag":"he_il",
-    "commonvoice_hours":1.4,
+    "commonvoice_hours":1.6,
     "commonvoice_locale":"he",
     "in_benchmark":true
   },
@@ -1375,7 +1375,7 @@
     "family":"Turkic",
     "flores_path":"uig_Arab",
     "fleurs_tag":null,
-    "commonvoice_hours":411.0,
+    "commonvoice_hours":412.0,
     "commonvoice_locale":"ug",
     "in_benchmark":true
   },
@@ -1747,7 +1747,7 @@
     "family":"Indo-European",
     "flores_path":"nob_Latn",
     "fleurs_tag":"nb_no",
-    "commonvoice_hours":0.5,
+    "commonvoice_hours":0.6,
     "commonvoice_locale":"nb-NO",
     "in_benchmark":true
   },
@@ -2167,7 +2167,7 @@
     "family":"Indo-European",
     "flores_path":"glg_Latn",
     "fleurs_tag":"gl_es",
-    "commonvoice_hours":117.0,
+    "commonvoice_hours":121.0,
     "commonvoice_locale":"gl",
     "in_benchmark":true
   },
@@ -2323,7 +2323,7 @@
     "family":"Dravidian",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":1.2,
+    "commonvoice_hours":11.0,
     "commonvoice_locale":"brh",
     "in_benchmark":false
   },
@@ -2623,7 +2623,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":0.9,
+    "commonvoice_hours":11.0,
     "commonvoice_locale":"haz",
     "in_benchmark":false
   },
@@ -2695,7 +2695,7 @@
     "family":"Indo-European",
     "flores_path":"oci_Latn",
     "fleurs_tag":"oc_fr",
-    "commonvoice_hours":1.8,
+    "commonvoice_hours":1.9,
     "commonvoice_locale":"oc",
     "in_benchmark":true
   },
@@ -3319,8 +3319,8 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"mfe",
     "in_benchmark":false
   },
   {
@@ -3331,7 +3331,7 @@
     "family":"Indo-European",
     "flores_path":"gle_Latn",
     "fleurs_tag":"ga_ie",
-    "commonvoice_hours":8.3,
+    "commonvoice_hours":8.8,
     "commonvoice_locale":"ga-IE",
     "in_benchmark":true
   },
@@ -3487,7 +3487,7 @@
     "family":"Indo-European",
     "flores_path":"lvs_Latn",
     "fleurs_tag":"lv_lv",
-    "commonvoice_hours":262.0,
+    "commonvoice_hours":263.0,
     "commonvoice_locale":"lv",
     "in_benchmark":true
   },
@@ -3535,7 +3535,7 @@
     "family":null,
     "flores_path":"eus_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":440.0,
+    "commonvoice_hours":452.0,
     "commonvoice_locale":"eu",
     "in_benchmark":true
   },
@@ -3559,7 +3559,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":83.0,
+    "commonvoice_hours":92.0,
     "commonvoice_locale":"kbd",
     "in_benchmark":false
   },
@@ -3679,7 +3679,7 @@
     "family":"Indo-European",
     "flores_path":"ydd_Hebr",
     "fleurs_tag":null,
-    "commonvoice_hours":0.7,
+    "commonvoice_hours":0.8,
     "commonvoice_locale":"yi",
     "in_benchmark":true
   },
@@ -3991,8 +3991,8 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":null,
-    "commonvoice_locale":null,
+    "commonvoice_hours":0.0,
+    "commonvoice_locale":"gaa",
     "in_benchmark":false
   },
   {
@@ -4351,7 +4351,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":29.0,
+    "commonvoice_hours":30.0,
     "commonvoice_locale":"br",
     "in_benchmark":false
   },
@@ -4651,7 +4651,7 @@
     "family":"Abkhaz-Adyge",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":30.0,
+    "commonvoice_hours":31.0,
     "commonvoice_locale":"ady",
     "in_benchmark":false
   },
@@ -7879,7 +7879,7 @@
     "family":"Artificial Language",
     "flores_path":"epo_Latn",
     "fleurs_tag":null,
-    "commonvoice_hours":1436.0,
+    "commonvoice_hours":1437.0,
     "commonvoice_locale":"eo",
     "in_benchmark":true
   },

models.json

@@ -28,7 +28,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1729555200000,
+    "creation_date":1729555200000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -48,7 +48,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1740355200000,
+    "creation_date":1740355200000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -68,7 +68,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1747872000000,
+    "creation_date":1747872000000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -83,7 +83,7 @@
     "id":"deepseek\/deepseek-chat",
     "name":"DeepSeek V3",
     "provider_name":"DeepSeek",
-    "cost":0.0,
+    "cost":0.27,
     "hf_id":"deepseek-ai\/DeepSeek-V3",
     "size":684531386000.0,
     "type":"open-source",
@@ -128,7 +128,7 @@
     "size":684531386000.0,
     "type":"open-source",
     "license":"Mit",
-    "creation_date":1737331200000,
+    "creation_date":1737331200000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -168,7 +168,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1738713600000,
+    "creation_date":1738713600000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -282,7 +282,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1750118400000,
+    "creation_date":1750118400000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -338,7 +338,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1715644800000,
+    "creation_date":1715644800000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -358,7 +358,7 @@
     "size":null,
     "type":"closed-source",
     "license":null,
-    "creation_date":1727913600000,
+    "creation_date":1727913600000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -413,7 +413,7 @@
     "size":null,
     "type":"open-source",
     "license":"Other",
-    "creation_date":1691625600000,
+    "creation_date":1691625600000.0,
     "tasks":[
       "translation_from",
       "translation_to",
@@ -512,7 +512,7 @@
     "id":"meta-llama\/llama-4-maverick",
     "name":"Llama 4 Maverick",
     "provider_name":"Meta",
-    "cost":0.0,
+    "cost":0.6,
     "hf_id":"meta-llama\/Llama-4-Maverick-17B-128E-Instruct",
     "size":401583781376.0,
     "type":"open-source",
@@ -772,7 +772,7 @@
     "id":"qwen\/qwen3-32b",
     "name":"Qwen3 32B",
     "provider_name":"Qwen",
-    "cost":0.0,
+    "cost":0.03,
     "hf_id":"Qwen\/Qwen3-32B",
     "size":32762123264.0,
     "type":"open-source",