Use standard Argon pricing for Pro leaderboard costs

README.md

@@ -33,6 +33,8 @@ Models and prices reflect each provider's official documentation; each system us
 
 Scores are percentages; costs are USD. The best score in each metric is **bold**; the second-best distinct score is <u>underlined</u>.
 
+Report costs using each provider's official public prices intended to remain in effect indefinitely, excluding discounts, free credits, and other promotions. Permanent changes to those prices change reported costs; temporary offers do not.
+
 <details>
 <summary><strong>Metric details</strong></summary>
 
@@ -102,7 +104,7 @@ Rows are ordered by % Perfect, then Value F1, then lower cost.
 | 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | <u>77.70</u> | 60.39 | 7.50¢ |
 | 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ |
 | 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ |
-| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.44 | 33.90 | 18.16¢ |
+| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.44 | 33.90 | 36.16¢ |
 | 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ |
 | 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ |
 | 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | <u>63.96</u> | 13.86¢ |

leaderboard-pro.csv

@@ -83,10 +83,10 @@ Datalab Accurate,Specialized APIs,house_counting_rules,12,9,0.00,45.78,0.00,0.00
 Datalab Accurate,Specialized APIs,drawn_relations,20,19,0.00,33.48,1.32,18.57,0.038200,0.00,33.48,1.32,18.57
 Datalab Accurate,Specialized APIs,charts_inside_documents,25,10,4.00,42.63,0.00,0.00,0.019400,4.00,42.63,0.00,0.00
 Datalab Accurate,Specialized APIs,long_chart_sheets,7,7,0.00,0.00,0.00,0.00,1.260000,0.00,0.00,0.00,0.00
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,all,100,81,30.00,42.44,33.90,35.81,0.181650,46.00,66.53,51.54,55.42
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,document_conventions,22,22,9.09,13.49,12.70,12.78,0.078295,27.27,45.07,42.01,42.57
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_vocabulary,14,14,50.00,50.00,31.24,35.19,0.046008,78.57,85.67,59.14,66.65
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_counting_rules,12,9,16.67,40.93,22.22,22.22,0.052237,16.67,44.20,25.93,25.93
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,drawn_relations,20,19,45.00,59.95,57.05,59.29,0.065557,55.00,79.77,70.70,72.94
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,charts_inside_documents,25,10,36.00,58.29,70.00,70.00,0.113076,56.00,85.47,80.00,80.00
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,long_chart_sheets,7,7,14.29,14.29,6.45,14.29,1.576212,28.57,28.57,6.55,28.57
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,all,100,81,30.00,42.44,33.90,35.81,0.361595,46.00,66.53,51.54,55.42
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,document_conventions,22,22,9.09,13.49,12.70,12.78,0.155289,27.27,45.07,42.01,42.57
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_vocabulary,14,14,50.00,50.00,31.24,35.19,0.091454,78.57,85.67,59.14,66.65
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_counting_rules,12,9,16.67,40.93,22.22,22.22,0.104433,16.67,44.20,25.93,25.93
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,drawn_relations,20,19,45.00,59.95,57.05,59.29,0.130325,55.00,79.77,70.70,72.94
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,charts_inside_documents,25,10,36.00,58.29,70.00,70.00,0.224565,56.00,85.47,80.00,80.00
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,long_chart_sheets,7,7,14.29,14.29,6.45,14.29,3.141275,28.57,28.57,6.55,28.57

src/extract_bench/utils/gemini_pricing.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
-from datetime import UTC, date, datetime
 from typing import Any
 
 # Gemini pricing: USD per million tokens (input, output).
@@ -71,10 +70,10 @@ def _longest_prefix_match(
     return max(matches, key=lambda item: len(item[0]))[1] if matches else default
 
 
-def cost_from_request_usage(model: str, usage: Mapping[str, Any], *, as_of: date | None = None) -> float | None:
+def cost_from_request_usage(model: str, usage: Mapping[str, Any]) -> float | None:
     """Price native CLI requests, including hidden thinking; unsupported accounting stays unknown.
 
-    Argon introductory rates end 2026-11-30:
+    Benchmark estimates use standard rates, excluding introductory discounts:
     https://ai.google.dev/gemini-api/docs/latest-model
     The CLI's Flash and Pro helpers use the shared standard Gemini pricing maps.
     """
@@ -87,8 +86,7 @@ def cost_from_request_usage(model: str, usage: Mapping[str, Any], *, as_of: date
     if usage.get("toolUsePromptTokenCount", 0) != 0:
         return None
     if model.startswith("gemini-4-argon"):
-        day = datetime.now(UTC).date() if as_of is None else as_of
-        input_rate, output_rate, cache_rate = (2.0, 10.0, 0.1) if day < date(2026, 12, 1) else (4.0, 20.0, 0.2)
+        input_rate, output_rate, cache_rate = (4.0, 20.0, 0.2)
     elif model.startswith(("gemini-3-flash-preview", "gemini-3.1-pro-preview")):
         input_rate, output_rate = gemini_pricing_per_million(model)
         cache_rate, _ = gemini_context_cache_pricing_per_million(model)

tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py

@@ -3,7 +3,6 @@
 import json
 import sys
 from collections import deque
-from datetime import date
 from pathlib import Path
 
 import pytest
@@ -31,7 +30,7 @@
                 "candidatesTokenCount": 500,
                 "totalTokenCount": 1643,
             },
-            0.00767,
+            0.01534,
         ),
         ("gemini-3.1-pro-preview-customtools", {"promptTokenCount": 200_000, "totalTokenCount": 200_100}, 0.4012),
         ("gemini-3.1-pro-preview-customtools", {"promptTokenCount": 200_001, "totalTokenCount": 200_101}, 0.801804),
@@ -46,7 +45,7 @@
 )
 def test_native_pricing_includes_thinking_and_context_tiers_and_preserves_unknown(model, usage, expected) -> None:
     """Visible output alone underprices thinking; absent usage and unknown rates must not become zero."""
-    cost = cost_from_request_usage(model, usage, as_of=date(2026, 10, 21))
+    cost = cost_from_request_usage(model, usage)
     assert cost is None if expected is None else cost == pytest.approx(expected)
 
 
@@ -172,10 +171,3 @@ def upstream(request):
         assert "gateway-secret" not in journal.read_text()
     finally:
         runner.shutdown()
-
-
-def test_argon_introductory_pricing_expires_without_changing_historical_replay():
-    usage = {"promptTokenCount": 1000, "cachedContentTokenCount": 400, "totalTokenCount": 1200}
-    intro = cost_from_request_usage("gemini-4-argon", usage, as_of=date(2026, 11, 30))
-    standard = cost_from_request_usage("gemini-4-argon", usage, as_of=date(2026, 12, 1))
-    assert standard == pytest.approx(2 * intro)

tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py

@@ -4,8 +4,6 @@
 import concurrent.futures
 import json
 import threading
-from datetime import date
-from functools import partial
 from pathlib import Path
 
 import pytest
@@ -18,9 +16,7 @@
 
 from extract_bench.inference.providers.extract.gemini_cli_trace import GeminiCliTrace
 from extract_bench.inference.providers.extract.gemini_cli_transport import GeminiCliTransport
-from extract_bench.utils.gemini_pricing import cost_from_request_usage as price_request
-
-cost_from_request_usage = partial(price_request, as_of=date(2026, 10, 21))
+from extract_bench.utils.gemini_pricing import cost_from_request_usage
 
 
 class Stream(httpx.SyncByteStream):
@@ -101,13 +97,13 @@ def emit(event, **payload):
             assert usage_seen.wait(5)
             assert not future.done()
             accounting = [e for e in journal(trace) if e["event"] == "attempt.accounting"][-1]
-            assert accounting["cost_usd"] == pytest.approx(0.00324)
+            assert accounting["cost_usd"] == pytest.approx(0.00648)
             assert accounting["cost_complete"] is False
         finally:
             release.set()
         assert future.result(timeout=5).content == first + last
     proxy.raise_if_failed()
-    assert proxy.summary()["total_cost_usd"] == pytest.approx(0.00424)
+    assert proxy.summary()["total_cost_usd"] == pytest.approx(0.00848)
     assert requests[0].url == "http://gateway.test/genai/v1beta/models/gemini-4-argon:streamGenerateContent?alt=sse"
     assert requests[0].content == payload
     assert requests[0].headers["x-goog-api-key"] == "configured-key"
@@ -142,7 +138,7 @@ def upstream(request):
         client.post(url, json={})
     summary = proxy.summary()
     assert len(summary["requests"]) == 2
-    assert summary["known_cost_usd"] == pytest.approx(0.00748)
+    assert summary["known_cost_usd"] == pytest.approx(0.01496)
     assert summary["total_cost_usd"] is None
     assert [item["outcome"] for item in summary["requests"]] == ["error", "success"]
     assert sum(e["event"] == "model.request.started" for e in journal(trace)) == 2
@@ -212,7 +208,7 @@ def close(self):
             trace.cancel("per-file timeout")
         future.result(timeout=5)
     summary = proxy.summary()
-    assert summary["known_cost_usd"] == pytest.approx(0.00324)
+    assert summary["known_cost_usd"] == pytest.approx(0.00648)
     assert summary["total_cost_usd"] is None
     saved = journal(trace)
     assert [e["reason"] for e in saved if e["event"] == "attempt.cancelled"] == ["per-file timeout"]
@@ -236,7 +232,7 @@ def test_error_frame_after_usage_does_not_claim_complete_cost(tmp_path: Path) ->
             client.post(proxy.cli_base_url + "/v1beta/models/gemini-4-argon:streamGenerateContent", json={}).content
             == payload
         )
-    assert proxy.summary()["known_cost_usd"] == pytest.approx(0.00324)
+    assert proxy.summary()["known_cost_usd"] == pytest.approx(0.00648)
     assert proxy.summary()["total_cost_usd"] is None
     assert proxy.summary()["requests"][0]["outcome"] == "response_error"
 
@@ -370,6 +366,6 @@ def closed_client(*args, **kwargs):
     assert not cancelled.is_set()
     summary = proxy.summary()
     assert len(summary["requests"]) == 2
-    assert summary["known_cost_usd"] == pytest.approx(0.00324)
+    assert summary["known_cost_usd"] == pytest.approx(0.00648)
     assert summary["total_cost_usd"] is None
     assert sum(e["event"] == "model.request.finished" for e in journal(trace)) == 2