diff --git a/README.md b/README.md
index 2f25cdd3..d1d223d5 100644
--- a/README.md
+++ b/README.md
@@ -33,6 +33,8 @@ Models and prices reflect each provider's official documentation; each system us
Scores are percentages; costs are USD. The best score in each metric is **bold**; the second-best distinct score is underlined.
+Report costs using each provider's official public prices intended to remain in effect indefinitely, excluding discounts, free credits, and other promotions. Permanent changes to those prices change reported costs; temporary offers do not.
+
Metric details
@@ -102,7 +104,7 @@ Rows are ordered by % Perfect, then Value F1, then lower cost.
| 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | 77.70 | 60.39 | 7.50¢ |
| 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ |
| 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ |
-| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.44 | 33.90 | 18.16¢ |
+| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.44 | 33.90 | 36.16¢ |
| 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ |
| 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ |
| 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | 63.96 | 13.86¢ |
diff --git a/leaderboard-pro.csv b/leaderboard-pro.csv
index aabfdf6b..0cb5671b 100644
--- a/leaderboard-pro.csv
+++ b/leaderboard-pro.csv
@@ -83,10 +83,10 @@ Datalab Accurate,Specialized APIs,house_counting_rules,12,9,0.00,45.78,0.00,0.00
Datalab Accurate,Specialized APIs,drawn_relations,20,19,0.00,33.48,1.32,18.57,0.038200,0.00,33.48,1.32,18.57
Datalab Accurate,Specialized APIs,charts_inside_documents,25,10,4.00,42.63,0.00,0.00,0.019400,4.00,42.63,0.00,0.00
Datalab Accurate,Specialized APIs,long_chart_sheets,7,7,0.00,0.00,0.00,0.00,1.260000,0.00,0.00,0.00,0.00
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,all,100,81,30.00,42.44,33.90,35.81,0.181650,46.00,66.53,51.54,55.42
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,document_conventions,22,22,9.09,13.49,12.70,12.78,0.078295,27.27,45.07,42.01,42.57
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_vocabulary,14,14,50.00,50.00,31.24,35.19,0.046008,78.57,85.67,59.14,66.65
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_counting_rules,12,9,16.67,40.93,22.22,22.22,0.052237,16.67,44.20,25.93,25.93
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,drawn_relations,20,19,45.00,59.95,57.05,59.29,0.065557,55.00,79.77,70.70,72.94
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,charts_inside_documents,25,10,36.00,58.29,70.00,70.00,0.113076,56.00,85.47,80.00,80.00
-Gemini CLI (Gemini 4 Argon Medium),Coding Agents,long_chart_sheets,7,7,14.29,14.29,6.45,14.29,1.576212,28.57,28.57,6.55,28.57
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,all,100,81,30.00,42.44,33.90,35.81,0.361595,46.00,66.53,51.54,55.42
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,document_conventions,22,22,9.09,13.49,12.70,12.78,0.155289,27.27,45.07,42.01,42.57
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_vocabulary,14,14,50.00,50.00,31.24,35.19,0.091454,78.57,85.67,59.14,66.65
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_counting_rules,12,9,16.67,40.93,22.22,22.22,0.104433,16.67,44.20,25.93,25.93
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,drawn_relations,20,19,45.00,59.95,57.05,59.29,0.130325,55.00,79.77,70.70,72.94
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,charts_inside_documents,25,10,36.00,58.29,70.00,70.00,0.224565,56.00,85.47,80.00,80.00
+Gemini CLI (Gemini 4 Argon Medium),Coding Agents,long_chart_sheets,7,7,14.29,14.29,6.45,14.29,3.141275,28.57,28.57,6.55,28.57
diff --git a/src/extract_bench/utils/gemini_pricing.py b/src/extract_bench/utils/gemini_pricing.py
index ba1376bf..b01e59d3 100644
--- a/src/extract_bench/utils/gemini_pricing.py
+++ b/src/extract_bench/utils/gemini_pricing.py
@@ -3,7 +3,6 @@
from __future__ import annotations
from collections.abc import Mapping
-from datetime import UTC, date, datetime
from typing import Any
# Gemini pricing: USD per million tokens (input, output).
@@ -71,10 +70,10 @@ def _longest_prefix_match(
return max(matches, key=lambda item: len(item[0]))[1] if matches else default
-def cost_from_request_usage(model: str, usage: Mapping[str, Any], *, as_of: date | None = None) -> float | None:
+def cost_from_request_usage(model: str, usage: Mapping[str, Any]) -> float | None:
"""Price native CLI requests, including hidden thinking; unsupported accounting stays unknown.
- Argon introductory rates end 2026-11-30:
+ Benchmark estimates use standard rates, excluding introductory discounts:
https://ai.google.dev/gemini-api/docs/latest-model
The CLI's Flash and Pro helpers use the shared standard Gemini pricing maps.
"""
@@ -87,8 +86,7 @@ def cost_from_request_usage(model: str, usage: Mapping[str, Any], *, as_of: date
if usage.get("toolUsePromptTokenCount", 0) != 0:
return None
if model.startswith("gemini-4-argon"):
- day = datetime.now(UTC).date() if as_of is None else as_of
- input_rate, output_rate, cache_rate = (2.0, 10.0, 0.1) if day < date(2026, 12, 1) else (4.0, 20.0, 0.2)
+ input_rate, output_rate, cache_rate = (4.0, 20.0, 0.2)
elif model.startswith(("gemini-3-flash-preview", "gemini-3.1-pro-preview")):
input_rate, output_rate = gemini_pricing_per_million(model)
cache_rate, _ = gemini_context_cache_pricing_per_million(model)
diff --git a/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py b/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py
index 0026a610..e8bb3338 100644
--- a/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py
+++ b/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py
@@ -3,7 +3,6 @@
import json
import sys
from collections import deque
-from datetime import date
from pathlib import Path
import pytest
@@ -31,7 +30,7 @@
"candidatesTokenCount": 500,
"totalTokenCount": 1643,
},
- 0.00767,
+ 0.01534,
),
("gemini-3.1-pro-preview-customtools", {"promptTokenCount": 200_000, "totalTokenCount": 200_100}, 0.4012),
("gemini-3.1-pro-preview-customtools", {"promptTokenCount": 200_001, "totalTokenCount": 200_101}, 0.801804),
@@ -46,7 +45,7 @@
)
def test_native_pricing_includes_thinking_and_context_tiers_and_preserves_unknown(model, usage, expected) -> None:
"""Visible output alone underprices thinking; absent usage and unknown rates must not become zero."""
- cost = cost_from_request_usage(model, usage, as_of=date(2026, 10, 21))
+ cost = cost_from_request_usage(model, usage)
assert cost is None if expected is None else cost == pytest.approx(expected)
@@ -172,10 +171,3 @@ def upstream(request):
assert "gateway-secret" not in journal.read_text()
finally:
runner.shutdown()
-
-
-def test_argon_introductory_pricing_expires_without_changing_historical_replay():
- usage = {"promptTokenCount": 1000, "cachedContentTokenCount": 400, "totalTokenCount": 1200}
- intro = cost_from_request_usage("gemini-4-argon", usage, as_of=date(2026, 11, 30))
- standard = cost_from_request_usage("gemini-4-argon", usage, as_of=date(2026, 12, 1))
- assert standard == pytest.approx(2 * intro)
diff --git a/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py b/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py
index b641fe0e..769b524b 100644
--- a/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py
+++ b/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py
@@ -4,8 +4,6 @@
import concurrent.futures
import json
import threading
-from datetime import date
-from functools import partial
from pathlib import Path
import pytest
@@ -18,9 +16,7 @@
from extract_bench.inference.providers.extract.gemini_cli_trace import GeminiCliTrace
from extract_bench.inference.providers.extract.gemini_cli_transport import GeminiCliTransport
-from extract_bench.utils.gemini_pricing import cost_from_request_usage as price_request
-
-cost_from_request_usage = partial(price_request, as_of=date(2026, 10, 21))
+from extract_bench.utils.gemini_pricing import cost_from_request_usage
class Stream(httpx.SyncByteStream):
@@ -101,13 +97,13 @@ def emit(event, **payload):
assert usage_seen.wait(5)
assert not future.done()
accounting = [e for e in journal(trace) if e["event"] == "attempt.accounting"][-1]
- assert accounting["cost_usd"] == pytest.approx(0.00324)
+ assert accounting["cost_usd"] == pytest.approx(0.00648)
assert accounting["cost_complete"] is False
finally:
release.set()
assert future.result(timeout=5).content == first + last
proxy.raise_if_failed()
- assert proxy.summary()["total_cost_usd"] == pytest.approx(0.00424)
+ assert proxy.summary()["total_cost_usd"] == pytest.approx(0.00848)
assert requests[0].url == "http://gateway.test/genai/v1beta/models/gemini-4-argon:streamGenerateContent?alt=sse"
assert requests[0].content == payload
assert requests[0].headers["x-goog-api-key"] == "configured-key"
@@ -142,7 +138,7 @@ def upstream(request):
client.post(url, json={})
summary = proxy.summary()
assert len(summary["requests"]) == 2
- assert summary["known_cost_usd"] == pytest.approx(0.00748)
+ assert summary["known_cost_usd"] == pytest.approx(0.01496)
assert summary["total_cost_usd"] is None
assert [item["outcome"] for item in summary["requests"]] == ["error", "success"]
assert sum(e["event"] == "model.request.started" for e in journal(trace)) == 2
@@ -212,7 +208,7 @@ def close(self):
trace.cancel("per-file timeout")
future.result(timeout=5)
summary = proxy.summary()
- assert summary["known_cost_usd"] == pytest.approx(0.00324)
+ assert summary["known_cost_usd"] == pytest.approx(0.00648)
assert summary["total_cost_usd"] is None
saved = journal(trace)
assert [e["reason"] for e in saved if e["event"] == "attempt.cancelled"] == ["per-file timeout"]
@@ -236,7 +232,7 @@ def test_error_frame_after_usage_does_not_claim_complete_cost(tmp_path: Path) ->
client.post(proxy.cli_base_url + "/v1beta/models/gemini-4-argon:streamGenerateContent", json={}).content
== payload
)
- assert proxy.summary()["known_cost_usd"] == pytest.approx(0.00324)
+ assert proxy.summary()["known_cost_usd"] == pytest.approx(0.00648)
assert proxy.summary()["total_cost_usd"] is None
assert proxy.summary()["requests"][0]["outcome"] == "response_error"
@@ -370,6 +366,6 @@ def closed_client(*args, **kwargs):
assert not cancelled.is_set()
summary = proxy.summary()
assert len(summary["requests"]) == 2
- assert summary["known_cost_usd"] == pytest.approx(0.00324)
+ assert summary["known_cost_usd"] == pytest.approx(0.00648)
assert summary["total_cost_usd"] is None
assert sum(e["event"] == "model.request.finished" for e in journal(trace)) == 2