Block gemini-2.5-pro-exp-03-25

evals/main.py

@@ -10,7 +10,7 @@ from tasks import tasks
 # ===== config =====
 
 n_sentences = 10
-n_languages = 40
+n_languages = 20
 n_models = 25
 
 # ===== run evaluation and aggregate results =====

evals/models.py

@@ -44,6 +44,10 @@ models = [
     "amazon/nova-micro-v1",  # 0.09$
 ]
 
+blocklist = [
+    "google/gemini-2.5-pro-exp-03-25" # rate limit too low
+]
+
 transcription_models = [
     "elevenlabs/scribe_v1",
     "openai/whisper-large-v3",
@@ -58,7 +62,6 @@ cache = Memory(location=".cache", verbose=0).cache
 def get_models(date: date):
     return get("https://openrouter.ai/api/frontend/models").json()["data"]
 
-
 def get_slug(permaslug):
     models = get_models(date.today())
     slugs = [m["slug"] for m in models if m["permaslug"] == permaslug]
@@ -88,12 +91,11 @@ def get_current_popular_models(date: date):
     data = sorted(data, key=lambda x: x["total_prompt_tokens"], reverse=True)
     return [get_slug(model["model_permaslug"]) for model in data]
 
-
 models += [
-    m for m in get_historical_popular_models(date.today()) if m and m not in models
+    m for m in get_historical_popular_models(date.today()) if m and m not in models and m not in blocklist
 ][:5]
 models += [
-    m for m in get_current_popular_models(date.today()) if m and m not in models
+    m for m in get_current_popular_models(date.today()) if m and m not in models and m not in blocklist
 ][:5]
 
 

languages.json

@@ -1027,7 +1027,7 @@
     "family":"Uralic",
     "flores_path":"hun_Latn",
     "fleurs_tag":"hu_hu",
-    "commonvoice_hours":92.0,
+    "commonvoice_hours":93.0,
     "commonvoice_locale":"hu",
     "in_benchmark":true
   },
@@ -2359,7 +2359,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":4.1,
+    "commonvoice_hours":4.3,
     "commonvoice_locale":"ibb",
     "in_benchmark":false
   },
@@ -5347,7 +5347,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":2.3,
+    "commonvoice_hours":2.6,
     "commonvoice_locale":"mua",
     "in_benchmark":false
   },
@@ -6199,7 +6199,7 @@
     "family":"Atlantic-Congo",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":3.9,
+    "commonvoice_hours":4.9,
     "commonvoice_locale":"jgo",
     "in_benchmark":false
   },
@@ -6367,7 +6367,7 @@
     "family":"Indo-European",
     "flores_path":null,
     "fleurs_tag":null,
-    "commonvoice_hours":0.6,
+    "commonvoice_hours":0.8,
     "commonvoice_locale":"btv",
     "in_benchmark":false
   },

models.json

@@ -218,5 +218,71 @@
     "type":"Commercial",
     "license":null,
     "creation_date":1733356800000
+  },
+  {
+    "id":"google\/gemini-2.0-flash-001",
+    "name":"Gemini 2.0 Flash",
+    "provider_name":"Google",
+    "cost":0.4,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1738713600000
+  },
+  {
+    "id":"google\/gemini-flash-1.5",
+    "name":"Gemini 1.5 Flash ",
+    "provider_name":"Google",
+    "cost":0.3,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1715644800000
+  },
+  {
+    "id":"google\/gemini-flash-1.5-8b",
+    "name":"Gemini 1.5 Flash 8B",
+    "provider_name":"Google",
+    "cost":0.15,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1727913600000
+  },
+  {
+    "id":"gryphe\/mythomax-l2-13b",
+    "name":"MythoMax 13B",
+    "provider_name":"MythoMax 13B",
+    "cost":0.07,
+    "hf_id":"Gryphe\/MythoMax-L2-13b",
+    "size":null,
+    "type":"Open",
+    "license":"Other",
+    "creation_date":1691625600000
+  },
+  {
+    "id":"microsoft\/wizardlm-2-8x22b",
+    "name":"WizardLM-2 8x22B",
+    "provider_name":"WizardLM-2 8x22B",
+    "cost":0.5,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1713225600000
+  },
+  {
+    "id":"x-ai\/grok-3-mini-beta",
+    "name":"Grok 3 Mini Beta",
+    "provider_name":"xAI",
+    "cost":0.5,
+    "hf_id":null,
+    "size":null,
+    "type":"Commercial",
+    "license":null,
+    "creation_date":1744156800000
   }
 ]