From 60cde454f6c096e0ff392d26f3e0becd343cf4d9 Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 22 Oct 2026 16:46:05 -0400 Subject: [PATCH 1/2] Use standard Argon pricing for Pro benchmark costs --- README.md | 4 ++-- leaderboard-pro.csv | 14 +++++++------- src/extract_bench/utils/gemini_pricing.py | 8 +++----- .../extract/test_gemini_cli_extract.py | 12 ++---------- .../extract/test_gemini_cli_transport.py | 18 +++++++----------- 5 files changed, 21 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 2f25cdd3..a3cbcdcd 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ This repository hosts two benchmarks. They share the task, the evaluator, the me Models and prices reflect each provider's official documentation; each system uses its recommended configuration. -Scores are percentages; costs are USD. The best score in each metric is **bold**; the second-best distinct score is underlined. +Scores are percentages; costs are USD at standard public rates, excluding introductory discounts. The best score in each metric is **bold**; the second-best distinct score is underlined.
Metric details @@ -102,7 +102,7 @@ Rows are ordered by % Perfect, then Value F1, then lower cost. | 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | 77.70 | 60.39 | 7.50¢ | | 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ | | 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ | -| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.44 | 33.90 | 18.16¢ | +| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.44 | 33.90 | 36.16¢ | | 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ | | 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ | | 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | 63.96 | 13.86¢ | diff --git a/leaderboard-pro.csv b/leaderboard-pro.csv index aabfdf6b..0cb5671b 100644 --- a/leaderboard-pro.csv +++ b/leaderboard-pro.csv @@ -83,10 +83,10 @@ Datalab Accurate,Specialized APIs,house_counting_rules,12,9,0.00,45.78,0.00,0.00 Datalab Accurate,Specialized APIs,drawn_relations,20,19,0.00,33.48,1.32,18.57,0.038200,0.00,33.48,1.32,18.57 Datalab Accurate,Specialized APIs,charts_inside_documents,25,10,4.00,42.63,0.00,0.00,0.019400,4.00,42.63,0.00,0.00 Datalab Accurate,Specialized APIs,long_chart_sheets,7,7,0.00,0.00,0.00,0.00,1.260000,0.00,0.00,0.00,0.00 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,all,100,81,30.00,42.44,33.90,35.81,0.181650,46.00,66.53,51.54,55.42 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,document_conventions,22,22,9.09,13.49,12.70,12.78,0.078295,27.27,45.07,42.01,42.57 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_vocabulary,14,14,50.00,50.00,31.24,35.19,0.046008,78.57,85.67,59.14,66.65 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_counting_rules,12,9,16.67,40.93,22.22,22.22,0.052237,16.67,44.20,25.93,25.93 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,drawn_relations,20,19,45.00,59.95,57.05,59.29,0.065557,55.00,79.77,70.70,72.94 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,charts_inside_documents,25,10,36.00,58.29,70.00,70.00,0.113076,56.00,85.47,80.00,80.00 -Gemini CLI (Gemini 4 Argon Medium),Coding Agents,long_chart_sheets,7,7,14.29,14.29,6.45,14.29,1.576212,28.57,28.57,6.55,28.57 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,all,100,81,30.00,42.44,33.90,35.81,0.361595,46.00,66.53,51.54,55.42 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,document_conventions,22,22,9.09,13.49,12.70,12.78,0.155289,27.27,45.07,42.01,42.57 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_vocabulary,14,14,50.00,50.00,31.24,35.19,0.091454,78.57,85.67,59.14,66.65 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,house_counting_rules,12,9,16.67,40.93,22.22,22.22,0.104433,16.67,44.20,25.93,25.93 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,drawn_relations,20,19,45.00,59.95,57.05,59.29,0.130325,55.00,79.77,70.70,72.94 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,charts_inside_documents,25,10,36.00,58.29,70.00,70.00,0.224565,56.00,85.47,80.00,80.00 +Gemini CLI (Gemini 4 Argon Medium),Coding Agents,long_chart_sheets,7,7,14.29,14.29,6.45,14.29,3.141275,28.57,28.57,6.55,28.57 diff --git a/src/extract_bench/utils/gemini_pricing.py b/src/extract_bench/utils/gemini_pricing.py index ba1376bf..b01e59d3 100644 --- a/src/extract_bench/utils/gemini_pricing.py +++ b/src/extract_bench/utils/gemini_pricing.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Mapping -from datetime import UTC, date, datetime from typing import Any # Gemini pricing: USD per million tokens (input, output). @@ -71,10 +70,10 @@ def _longest_prefix_match( return max(matches, key=lambda item: len(item[0]))[1] if matches else default -def cost_from_request_usage(model: str, usage: Mapping[str, Any], *, as_of: date | None = None) -> float | None: +def cost_from_request_usage(model: str, usage: Mapping[str, Any]) -> float | None: """Price native CLI requests, including hidden thinking; unsupported accounting stays unknown. - Argon introductory rates end 2026-11-30: + Benchmark estimates use standard rates, excluding introductory discounts: https://ai.google.dev/gemini-api/docs/latest-model The CLI's Flash and Pro helpers use the shared standard Gemini pricing maps. """ @@ -87,8 +86,7 @@ def cost_from_request_usage(model: str, usage: Mapping[str, Any], *, as_of: date if usage.get("toolUsePromptTokenCount", 0) != 0: return None if model.startswith("gemini-4-argon"): - day = datetime.now(UTC).date() if as_of is None else as_of - input_rate, output_rate, cache_rate = (2.0, 10.0, 0.1) if day < date(2026, 12, 1) else (4.0, 20.0, 0.2) + input_rate, output_rate, cache_rate = (4.0, 20.0, 0.2) elif model.startswith(("gemini-3-flash-preview", "gemini-3.1-pro-preview")): input_rate, output_rate = gemini_pricing_per_million(model) cache_rate, _ = gemini_context_cache_pricing_per_million(model) diff --git a/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py b/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py index 0026a610..e8bb3338 100644 --- a/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py +++ b/tests/extract_bench/inference/providers/extract/test_gemini_cli_extract.py @@ -3,7 +3,6 @@ import json import sys from collections import deque -from datetime import date from pathlib import Path import pytest @@ -31,7 +30,7 @@ "candidatesTokenCount": 500, "totalTokenCount": 1643, }, - 0.00767, + 0.01534, ), ("gemini-3.1-pro-preview-customtools", {"promptTokenCount": 200_000, "totalTokenCount": 200_100}, 0.4012), ("gemini-3.1-pro-preview-customtools", {"promptTokenCount": 200_001, "totalTokenCount": 200_101}, 0.801804), @@ -46,7 +45,7 @@ ) def test_native_pricing_includes_thinking_and_context_tiers_and_preserves_unknown(model, usage, expected) -> None: """Visible output alone underprices thinking; absent usage and unknown rates must not become zero.""" - cost = cost_from_request_usage(model, usage, as_of=date(2026, 10, 21)) + cost = cost_from_request_usage(model, usage) assert cost is None if expected is None else cost == pytest.approx(expected) @@ -172,10 +171,3 @@ def upstream(request): assert "gateway-secret" not in journal.read_text() finally: runner.shutdown() - - -def test_argon_introductory_pricing_expires_without_changing_historical_replay(): - usage = {"promptTokenCount": 1000, "cachedContentTokenCount": 400, "totalTokenCount": 1200} - intro = cost_from_request_usage("gemini-4-argon", usage, as_of=date(2026, 11, 30)) - standard = cost_from_request_usage("gemini-4-argon", usage, as_of=date(2026, 12, 1)) - assert standard == pytest.approx(2 * intro) diff --git a/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py b/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py index b641fe0e..769b524b 100644 --- a/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py +++ b/tests/extract_bench/inference/providers/extract/test_gemini_cli_transport.py @@ -4,8 +4,6 @@ import concurrent.futures import json import threading -from datetime import date -from functools import partial from pathlib import Path import pytest @@ -18,9 +16,7 @@ from extract_bench.inference.providers.extract.gemini_cli_trace import GeminiCliTrace from extract_bench.inference.providers.extract.gemini_cli_transport import GeminiCliTransport -from extract_bench.utils.gemini_pricing import cost_from_request_usage as price_request - -cost_from_request_usage = partial(price_request, as_of=date(2026, 10, 21)) +from extract_bench.utils.gemini_pricing import cost_from_request_usage class Stream(httpx.SyncByteStream): @@ -101,13 +97,13 @@ def emit(event, **payload): assert usage_seen.wait(5) assert not future.done() accounting = [e for e in journal(trace) if e["event"] == "attempt.accounting"][-1] - assert accounting["cost_usd"] == pytest.approx(0.00324) + assert accounting["cost_usd"] == pytest.approx(0.00648) assert accounting["cost_complete"] is False finally: release.set() assert future.result(timeout=5).content == first + last proxy.raise_if_failed() - assert proxy.summary()["total_cost_usd"] == pytest.approx(0.00424) + assert proxy.summary()["total_cost_usd"] == pytest.approx(0.00848) assert requests[0].url == "http://gateway.test/genai/v1beta/models/gemini-4-argon:streamGenerateContent?alt=sse" assert requests[0].content == payload assert requests[0].headers["x-goog-api-key"] == "configured-key" @@ -142,7 +138,7 @@ def upstream(request): client.post(url, json={}) summary = proxy.summary() assert len(summary["requests"]) == 2 - assert summary["known_cost_usd"] == pytest.approx(0.00748) + assert summary["known_cost_usd"] == pytest.approx(0.01496) assert summary["total_cost_usd"] is None assert [item["outcome"] for item in summary["requests"]] == ["error", "success"] assert sum(e["event"] == "model.request.started" for e in journal(trace)) == 2 @@ -212,7 +208,7 @@ def close(self): trace.cancel("per-file timeout") future.result(timeout=5) summary = proxy.summary() - assert summary["known_cost_usd"] == pytest.approx(0.00324) + assert summary["known_cost_usd"] == pytest.approx(0.00648) assert summary["total_cost_usd"] is None saved = journal(trace) assert [e["reason"] for e in saved if e["event"] == "attempt.cancelled"] == ["per-file timeout"] @@ -236,7 +232,7 @@ def test_error_frame_after_usage_does_not_claim_complete_cost(tmp_path: Path) -> client.post(proxy.cli_base_url + "/v1beta/models/gemini-4-argon:streamGenerateContent", json={}).content == payload ) - assert proxy.summary()["known_cost_usd"] == pytest.approx(0.00324) + assert proxy.summary()["known_cost_usd"] == pytest.approx(0.00648) assert proxy.summary()["total_cost_usd"] is None assert proxy.summary()["requests"][0]["outcome"] == "response_error" @@ -370,6 +366,6 @@ def closed_client(*args, **kwargs): assert not cancelled.is_set() summary = proxy.summary() assert len(summary["requests"]) == 2 - assert summary["known_cost_usd"] == pytest.approx(0.00324) + assert summary["known_cost_usd"] == pytest.approx(0.00648) assert summary["total_cost_usd"] is None assert sum(e["event"] == "model.request.finished" for e in journal(trace)) == 2 From 9a1671039b4d3a2e858c3a4da291ae116e7a3eb0 Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 22 Oct 2026 16:50:53 -0400 Subject: [PATCH 2/2] Clarify ongoing public pricing policy --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a3cbcdcd..d1d223d5 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,9 @@ This repository hosts two benchmarks. They share the task, the evaluator, the me Models and prices reflect each provider's official documentation; each system uses its recommended configuration. -Scores are percentages; costs are USD at standard public rates, excluding introductory discounts. The best score in each metric is **bold**; the second-best distinct score is underlined. +Scores are percentages; costs are USD. The best score in each metric is **bold**; the second-best distinct score is underlined. + +Report costs using each provider's official public prices intended to remain in effect indefinitely, excluding discounts, free credits, and other promotions. Permanent changes to those prices change reported costs; temporary offers do not.
Metric details