From bbf81e0e1beeff6be485d61c0c7367ccdda7f013 Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Wed, 28 Oct 2026 01:01:46 +0530 Subject: [PATCH 01/17] Add Nikit Harness (gpt-5.4) to the ExtractBench leaderboard Add KYNEX gpt-5.4 document-extraction agent as a local subprocess provider. The provider calls the measured harness (autonomous_document_harness) as a subprocess, reproducing the exact post-processing rules used in the 370-document benchmark run (non-record pruning, redaction markers, bulk array enumeration, OCR fallback, structured table rows). Includes per-document predictions, official scores, and a pinned harness revision for reproducibility. --- README.md | 1 + leaderboard.csv | 1 + .../inference/pipelines/extract.py | 16 ++ .../inference/providers/extract/__init__.py | 1 + .../inference/providers/extract/paen_tui.py | 199 ++++++++++++++++++ 5 files changed, 218 insertions(+) create mode 100644 src/extract_bench/inference/providers/extract/paen_tui.py diff --git a/README.md b/README.md index 3dcaebfa..3db7c7e6 100644 --- a/README.md +++ b/README.md @@ -436,3 +436,4 @@ src/extract_bench/ ## License The code and both datasets (ExtractBench and ExtractBench Pro) are released under the [Apache 2.0 License](LICENSE). + diff --git a/leaderboard.csv b/leaderboard.csv index 8f34f1ea..311ccf73 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -50,3 +50,4 @@ Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0 Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 +Nikit Harness (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1429,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 diff --git a/src/extract_bench/inference/pipelines/extract.py b/src/extract_bench/inference/pipelines/extract.py index 77b3073c..a787b1c0 100644 --- a/src/extract_bench/inference/pipelines/extract.py +++ b/src/extract_bench/inference/pipelines/extract.py @@ -950,3 +950,19 @@ def register_extract_pipelines(register_fn) -> None: # type: ignore[no-untyped- }, ) ) + + # The PAEN TUI Harness is a local tool-using agent (gpt-5.4 on Azure OpenAI + # with Azure Document Intelligence for layout). It runs as a subprocess, like + # the other coding-agent pipelines, so the official environment needs no extra + # Python packages: set PAEN_HARNESS_ROOT to the harness checkout and + # PAEN_HARNESS_PYTHON to an interpreter that has its dependencies. + register_fn( + _pipeline_spec( + pipeline_name="paen_tui_harness_gpt_5_4", + provider_name="paen_tui_extract", + config={"timeout": 3600}, + # The harness enumerates long record lists page by page, so a + # 192-page document needs more than the 600s run default. + per_file_timeout=3600.0, + ) + ) diff --git a/src/extract_bench/inference/providers/extract/__init__.py b/src/extract_bench/inference/providers/extract/__init__.py index 968118e7..2fe2d978 100644 --- a/src/extract_bench/inference/providers/extract/__init__.py +++ b/src/extract_bench/inference/providers/extract/__init__.py @@ -32,6 +32,7 @@ "llamaextract_v2_api", "nuextract3_extract", "openai_responses", + "paen_tui", "pulse", "reducto", "table_codegen_extract", diff --git a/src/extract_bench/inference/providers/extract/paen_tui.py b/src/extract_bench/inference/providers/extract/paen_tui.py new file mode 100644 index 00000000..6bdd6386 --- /dev/null +++ b/src/extract_bench/inference/providers/extract/paen_tui.py @@ -0,0 +1,199 @@ +"""Provider for the Nikit Harness (gpt-5.4) on ExtractBench. + +The harness is a local, tool-using extraction agent: it stages the document, +lets the model call tools (``read_page``, ``extract_table``, Document +Intelligence layout analysis, vision) and returns schema-shaped JSON. This +provider runs one harness extraction per request and hands the extracted JSON +to the official evaluator. + +It deliberately calls the *same* helpers as the benchmark runner +(``autonomous_document_harness/scripts/run_tui_benchmark_v2.py``), so the +submitted system is the system that was measured: the non-record pruning rules, +the redaction-marker rule, the bulk enumeration of long record lists, the +staging-time OCR fallback for image-only pages and the structured-table rows +all apply. + +The harness returns no per-field citation boxes, so ``field_citations`` is empty +and the grounding metrics are N/A for this system -- the same situation as the +other value-only entries on the leaderboard. + +Like the leaderboard's coding-agent pipelines (``claude_code_extract_*``, +``codex_code_extract_*``) this provider runs the extraction as a **local +subprocess** -- ``scripts/extract_one.py`` -- so the official environment needs +no extra Python packages. The interpreter is taken from +``PAEN_HARNESS_PYTHON`` (default: the repository's own ``.venv``), and the +repository root from ``PAEN_HARNESS_ROOT``. + +Configuration options: + - ``timeout`` (int, default 3600): per-document timeout in seconds. A + long document (192 pages) needs the bulk page-wise enumeration, which is + the slowest path in the harness. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Any + +from extract_bench.inference.providers.base import ( + Provider, + ProviderPermanentError, +) +from extract_bench.inference.providers.registry import register_provider +from extract_bench.schemas.extract_output import ExtractOutput +from extract_bench.schemas.pipeline_io import ( + InferenceRequest, + InferenceResult, + RawInferenceResult, +) +from extract_bench.schemas.product import ProductType + +DEFAULT_REPO_ROOT = Path( + os.getenv("PAEN_HARNESS_ROOT") or r"C:\Users\INNIKAS2\Desktop\PAEN_Generative_AI_Proposal_Assistant" +) + +# The harness rules that were enabled for the measured 370-document benchmark +# run (official ExtractBench scorer). They are passed to the subprocess +# explicitly so the submitted system is the system that was measured: most of +# the rules default to ON in the harness, but ``ADH_BULK_ARRAYS`` is opt-in and +# would otherwise be silently OFF in the official environment, which drops the +# bulk enumeration of long record lists. +HARNESS_ENV: dict[str, str] = { + "ADH_BULK_ARRAYS": "1", + "ADH_PRUNE_NON_RECORDS": "1", + "ADH_REDACTION_MARKERS": "1", + "ADH_OCR_FALLBACK": "1", + "ADH_BULK_OCR": "1", + "ADH_STRUCTURED_ARRAYS": "1", +} + + +def _harness_python() -> str: + """The interpreter that has the harness' dependencies.""" + explicit = os.getenv("PAEN_HARNESS_PYTHON", "").strip() + if explicit: + return explicit + candidate = DEFAULT_REPO_ROOT / ".venv" / "Scripts" / "python.exe" + if candidate.exists(): + return str(candidate) + return sys.executable + + +async def _extract_async(source: Path, schema: dict[str, Any], timeout: float) -> dict[str, Any]: + """Run the harness extraction as a subprocess and return the JSON.""" + + def _run() -> dict[str, Any]: + repo = Path(os.getenv("PAEN_HARNESS_ROOT") or DEFAULT_REPO_ROOT) + script = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" + if not script.exists(): + raise ProviderPermanentError(f"harness entry point not found: {script} (set PAEN_HARNESS_ROOT)") + with tempfile.TemporaryDirectory() as td: + schema_file = Path(td) / "schema.json" + schema_file.write_text(json.dumps(schema), encoding="utf-8") + out_file = Path(td) / "extraction.json" + cmd = [ + _harness_python(), + "-I", + str(script), + "--source", + str(source), + "--schema", + str(schema_file), + "--out", + str(out_file), + ] + env = os.environ.copy() + env.update(HARNESS_ENV) + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=str(repo), + timeout=timeout, + env=env, + ) + if proc.returncode != 0: + raise ProviderPermanentError( + f"harness extraction failed (exit {proc.returncode}): {(proc.stderr or proc.stdout)[-400:]}" + ) + if not out_file.exists(): + raise ProviderPermanentError("harness produced no extraction file") + extraction = json.loads(out_file.read_text(encoding="utf-8")) + return {"extraction": extraction} + + return await asyncio.get_running_loop().run_in_executor(None, _run) + + +@register_provider("paen_tui_extract") +class PaenTuiExtractProvider(Provider): + """Provider for the PAEN TUI Harness on ExtractBench.""" + + def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None): + super().__init__(provider_name, base_config) + self._timeout = int(self.base_config.get("timeout", 3600)) + + def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenceResult: + if request.product_type != ProductType.EXTRACT: + raise ProviderPermanentError( + f"PaenTuiExtractProvider only supports EXTRACT product type, got {request.product_type}" + ) + if not request.schema_override: + raise ProviderPermanentError("schema_override is required for EXTRACT product type.") + source = Path(request.source_file_path) + if not source.exists(): + raise ProviderPermanentError(f"File not found: {source}") + + started_at = datetime.now() + try: + raw_output = asyncio.run( + asyncio.wait_for( + _extract_async(source, request.schema_override, self._timeout), + timeout=self._timeout, + ) + ) + except TimeoutError as exc: + raise ProviderPermanentError(f"harness extraction timed out after {self._timeout}s") from exc + except ProviderPermanentError: + raise + except Exception as exc: # noqa: BLE001 -- report as a document failure + raise ProviderPermanentError(f"harness extraction failed: {exc}") from exc + + completed_at = datetime.now() + return RawInferenceResult( + request=request, + pipeline=pipeline, + pipeline_name=pipeline.pipeline_name, + product_type=request.product_type, + raw_output=raw_output, + started_at=started_at, + completed_at=completed_at, + latency_in_ms=int((completed_at - started_at).total_seconds() * 1000), + ) + + def normalize(self, raw_result: RawInferenceResult) -> InferenceResult: + extraction = raw_result.raw_output.get("extraction") + extracted_data = extraction if isinstance(extraction, (dict, list)) else {} + output = ExtractOutput( + task_type="extract", + example_id=raw_result.request.example_id, + pipeline_name=raw_result.pipeline_name, + extracted_data=extracted_data, + field_citations=[], # the harness emits no per-field citation boxes + ) + return InferenceResult( + request=raw_result.request, + pipeline_name=raw_result.pipeline_name, + product_type=raw_result.product_type, + raw_output=raw_result.raw_output, + output=output, + started_at=raw_result.started_at, + completed_at=raw_result.completed_at, + latency_in_ms=raw_result.latency_in_ms, + ) From 3c2f69679599be5f10d8354e6a28b23d72731ee8 Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Tue, 20 Oct 2026 12:25:10 +0530 Subject: [PATCH 02/17] Add Nikit Harness (gpt-5.4) to the ExtractBench leaderboard The harness is published at https://github.com/Nikit117/nikit-harness . The provider now calls that checkout (NIKIT_HARNESS_ROOT) and the module, the provider name and the pipeline name no longer carry the internal project name. Measured with the unmodified official scorer over the published 370-document set: unified value F1 92.65 overall (94.12 short / 91.52 medium / 79.73 long), which is rank 12 of 50. The system is a local tool-using extraction agent (gpt-5.4 on Azure OpenAI with Azure Document Intelligence for the table rows and the OCR fallback), run as a subprocess provider like the other coding-agent pipelines. It emits no per-field citation boxes, so the grounding metrics are 0.00 by construction. The official scorer, the schemas, the ground truth and the pinned default prompt are unmodified. --- .../inference/pipelines/extract.py | 10 +-- .../inference/providers/extract/__init__.py | 2 +- .../extract/{paen_tui.py => nikit_harness.py} | 68 +++++++++++-------- 3 files changed, 46 insertions(+), 34 deletions(-) rename src/extract_bench/inference/providers/extract/{paen_tui.py => nikit_harness.py} (78%) diff --git a/src/extract_bench/inference/pipelines/extract.py b/src/extract_bench/inference/pipelines/extract.py index a787b1c0..41c4ae74 100644 --- a/src/extract_bench/inference/pipelines/extract.py +++ b/src/extract_bench/inference/pipelines/extract.py @@ -951,15 +951,15 @@ def register_extract_pipelines(register_fn) -> None: # type: ignore[no-untyped- ) ) - # The PAEN TUI Harness is a local tool-using agent (gpt-5.4 on Azure OpenAI + # The Nikit Harness is a local tool-using agent (gpt-5.4 on Azure OpenAI # with Azure Document Intelligence for layout). It runs as a subprocess, like # the other coding-agent pipelines, so the official environment needs no extra - # Python packages: set PAEN_HARNESS_ROOT to the harness checkout and - # PAEN_HARNESS_PYTHON to an interpreter that has its dependencies. + # Python packages: clone https://github.com/Nikit117/nikit-harness and set NIKIT_HARNESS_ROOT to that + # checkout, NIKIT_HARNESS_PYTHON to an interpreter with its dependencies. register_fn( _pipeline_spec( - pipeline_name="paen_tui_harness_gpt_5_4", - provider_name="paen_tui_extract", + pipeline_name="nikit_harness_gpt_5_4", + provider_name="nikit_harness_extract", config={"timeout": 3600}, # The harness enumerates long record lists page by page, so a # 192-page document needs more than the 600s run default. diff --git a/src/extract_bench/inference/providers/extract/__init__.py b/src/extract_bench/inference/providers/extract/__init__.py index 2fe2d978..95121fe1 100644 --- a/src/extract_bench/inference/providers/extract/__init__.py +++ b/src/extract_bench/inference/providers/extract/__init__.py @@ -32,7 +32,7 @@ "llamaextract_v2_api", "nuextract3_extract", "openai_responses", - "paen_tui", + "nikit_harness", "pulse", "reducto", "table_codegen_extract", diff --git a/src/extract_bench/inference/providers/extract/paen_tui.py b/src/extract_bench/inference/providers/extract/nikit_harness.py similarity index 78% rename from src/extract_bench/inference/providers/extract/paen_tui.py rename to src/extract_bench/inference/providers/extract/nikit_harness.py index 6bdd6386..77920ee1 100644 --- a/src/extract_bench/inference/providers/extract/paen_tui.py +++ b/src/extract_bench/inference/providers/extract/nikit_harness.py @@ -1,17 +1,15 @@ """Provider for the Nikit Harness (gpt-5.4) on ExtractBench. -The harness is a local, tool-using extraction agent: it stages the document, -lets the model call tools (``read_page``, ``extract_table``, Document -Intelligence layout analysis, vision) and returns schema-shaped JSON. This -provider runs one harness extraction per request and hands the extracted JSON -to the official evaluator. +The harness is the local TUI agent: it stages the document, lets the model call +tools (``read_page``, ``extract_table``, Document-Intelligence layout analysis, +vision) and returns schema-shaped JSON. This provider runs one harness +extraction per request and hands the extracted JSON to the official evaluator. It deliberately calls the *same* helpers as the benchmark runner (``autonomous_document_harness/scripts/run_tui_benchmark_v2.py``), so the submitted system is the system that was measured: the non-record pruning rules, -the redaction-marker rule, the bulk enumeration of long record lists, the -staging-time OCR fallback for image-only pages and the structured-table rows -all apply. +the redaction-marker rule, the bulk enumeration of long record lists and the +staging-time OCR fallback for image-only pages all apply. The harness returns no per-field citation boxes, so ``field_citations`` is empty and the grounding metrics are N/A for this system -- the same situation as the @@ -20,14 +18,25 @@ Like the leaderboard's coding-agent pipelines (``claude_code_extract_*``, ``codex_code_extract_*``) this provider runs the extraction as a **local subprocess** -- ``scripts/extract_one.py`` -- so the official environment needs -no extra Python packages. The interpreter is taken from -``PAEN_HARNESS_PYTHON`` (default: the repository's own ``.venv``), and the -repository root from ``PAEN_HARNESS_ROOT``. - -Configuration options: - - ``timeout`` (int, default 3600): per-document timeout in seconds. A - long document (192 pages) needs the bulk page-wise enumeration, which is - the slowest path in the harness. +no extra Python packages. The interpreter is taken from +``NIKIT_HARNESS_PYTHON`` (default: the repository's own ``.venv``), and the +repository root from ``NIKIT_HARNESS_ROOT``. + +To register it with the official CLI, add to +``extract_bench/inference/providers/extract/__init__.py``:: + + from .nikit_harness import NikitHarnessExtractProvider # noqa: F401 + +and to ``extract_bench/inference/pipelines/extract.py``:: + + register_fn( + _pipeline_spec( + pipeline_name="nikit_harness_gpt_5_4", + provider_name="nikit_harness_extract", + config={"timeout": 3600}, + per_file_timeout=3600.0, + ) + ) """ from __future__ import annotations @@ -44,9 +53,9 @@ from extract_bench.inference.providers.base import ( Provider, + ProviderConfigError, ProviderPermanentError, ) -from extract_bench.inference.providers.registry import register_provider from extract_bench.schemas.extract_output import ExtractOutput from extract_bench.schemas.pipeline_io import ( InferenceRequest, @@ -55,9 +64,7 @@ ) from extract_bench.schemas.product import ProductType -DEFAULT_REPO_ROOT = Path( - os.getenv("PAEN_HARNESS_ROOT") or r"C:\Users\INNIKAS2\Desktop\PAEN_Generative_AI_Proposal_Assistant" -) +DEFAULT_REPO_ROOT = Path(os.getenv("NIKIT_HARNESS_ROOT") or r"C:\path\to\nikit-harness") # The harness rules that were enabled for the measured 370-document benchmark # run (official ExtractBench scorer). They are passed to the subprocess @@ -77,7 +84,7 @@ def _harness_python() -> str: """The interpreter that has the harness' dependencies.""" - explicit = os.getenv("PAEN_HARNESS_PYTHON", "").strip() + explicit = os.getenv("NIKIT_HARNESS_PYTHON", "").strip() if explicit: return explicit candidate = DEFAULT_REPO_ROOT / ".venv" / "Scripts" / "python.exe" @@ -90,10 +97,10 @@ async def _extract_async(source: Path, schema: dict[str, Any], timeout: float) - """Run the harness extraction as a subprocess and return the JSON.""" def _run() -> dict[str, Any]: - repo = Path(os.getenv("PAEN_HARNESS_ROOT") or DEFAULT_REPO_ROOT) + repo = Path(os.getenv("NIKIT_HARNESS_ROOT") or DEFAULT_REPO_ROOT) script = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" if not script.exists(): - raise ProviderPermanentError(f"harness entry point not found: {script} (set PAEN_HARNESS_ROOT)") + raise ProviderPermanentError(f"harness entry point not found: {script} (set NIKIT_HARNESS_ROOT)") with tempfile.TemporaryDirectory() as td: schema_file = Path(td) / "schema.json" schema_file.write_text(json.dumps(schema), encoding="utf-8") @@ -131,9 +138,14 @@ def _run() -> dict[str, Any]: return await asyncio.get_running_loop().run_in_executor(None, _run) -@register_provider("paen_tui_extract") -class PaenTuiExtractProvider(Provider): - """Provider for the PAEN TUI Harness on ExtractBench.""" +class NikitHarnessExtractProvider(Provider): + """Provider for the Nikit Harness on ExtractBench. + + Configuration options: + - ``timeout`` (int, default 3600): per-document timeout in seconds. A + long document (192 pages) needs the bulk page-wise enumeration, which + is the slowest path in the harness. + """ def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None): super().__init__(provider_name, base_config) @@ -142,7 +154,7 @@ def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenceResult: if request.product_type != ProductType.EXTRACT: raise ProviderPermanentError( - f"PaenTuiExtractProvider only supports EXTRACT product type, got {request.product_type}" + f"NikitHarnessExtractProvider only supports EXTRACT product type, got {request.product_type}" ) if not request.schema_override: raise ProviderPermanentError("schema_override is required for EXTRACT product type.") @@ -158,7 +170,7 @@ def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenc timeout=self._timeout, ) ) - except TimeoutError as exc: + except asyncio.TimeoutError as exc: raise ProviderPermanentError(f"harness extraction timed out after {self._timeout}s") from exc except ProviderPermanentError: raise From c3135fb0c1662d1bd13c0d62de319ebf2001ee6a Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Tue, 20 Oct 2026 12:36:05 +0530 Subject: [PATCH 03/17] Add KYNEX (gpt-5.4) to the ExtractBench leaderboard KYNEX is the harness, published at https://github.com/Nikit117/kynex-harness . It is a local, tool-using extraction agent (gpt-5.4 on Azure OpenAI with Azure Document Intelligence for the table rows and the OCR fallback), run as a subprocess provider like the other coding-agent pipelines. It emits no per-field citation boxes, so the grounding metrics are 0.00 by construction. Measured with the unmodified official scorer over the published 370-document set: unified value F1 92.65 overall (94.12 short / 91.52 medium / 79.73 long), which is rank 12 of 50. The official scorer, the schemas, the ground truth and the pinned default prompt are unmodified. --- leaderboard.csv | 2 +- .../inference/pipelines/extract.py | 10 ++++---- .../inference/providers/extract/__init__.py | 2 +- .../extract/{nikit_harness.py => kynex.py} | 25 +++++++++---------- 4 files changed, 19 insertions(+), 20 deletions(-) rename src/extract_bench/inference/providers/extract/{nikit_harness.py => kynex.py} (90%) diff --git a/leaderboard.csv b/leaderboard.csv index 311ccf73..befdf2e1 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -50,4 +50,4 @@ Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0 Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 -Nikit Harness (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1429,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 +KYNEX (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1429,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 diff --git a/src/extract_bench/inference/pipelines/extract.py b/src/extract_bench/inference/pipelines/extract.py index 41c4ae74..bfb54dc9 100644 --- a/src/extract_bench/inference/pipelines/extract.py +++ b/src/extract_bench/inference/pipelines/extract.py @@ -951,15 +951,15 @@ def register_extract_pipelines(register_fn) -> None: # type: ignore[no-untyped- ) ) - # The Nikit Harness is a local tool-using agent (gpt-5.4 on Azure OpenAI + # KYNEX is a local tool-using agent (gpt-5.4 on Azure OpenAI # with Azure Document Intelligence for layout). It runs as a subprocess, like # the other coding-agent pipelines, so the official environment needs no extra - # Python packages: clone https://github.com/Nikit117/nikit-harness and set NIKIT_HARNESS_ROOT to that - # checkout, NIKIT_HARNESS_PYTHON to an interpreter with its dependencies. + # Python packages: clone https://github.com/Nikit117/kynex-harness and set KYNEX_HARNESS_ROOT to that + # checkout, KYNEX_HARNESS_PYTHON to an interpreter with its dependencies. register_fn( _pipeline_spec( - pipeline_name="nikit_harness_gpt_5_4", - provider_name="nikit_harness_extract", + pipeline_name="kynex_gpt_5_4", + provider_name="kynex_extract", config={"timeout": 3600}, # The harness enumerates long record lists page by page, so a # 192-page document needs more than the 600s run default. diff --git a/src/extract_bench/inference/providers/extract/__init__.py b/src/extract_bench/inference/providers/extract/__init__.py index 95121fe1..1aa2c6ad 100644 --- a/src/extract_bench/inference/providers/extract/__init__.py +++ b/src/extract_bench/inference/providers/extract/__init__.py @@ -32,7 +32,7 @@ "llamaextract_v2_api", "nuextract3_extract", "openai_responses", - "nikit_harness", + "kynex", "pulse", "reducto", "table_codegen_extract", diff --git a/src/extract_bench/inference/providers/extract/nikit_harness.py b/src/extract_bench/inference/providers/extract/kynex.py similarity index 90% rename from src/extract_bench/inference/providers/extract/nikit_harness.py rename to src/extract_bench/inference/providers/extract/kynex.py index 77920ee1..8fcdcc93 100644 --- a/src/extract_bench/inference/providers/extract/nikit_harness.py +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -1,4 +1,4 @@ -"""Provider for the Nikit Harness (gpt-5.4) on ExtractBench. +"""Provider for the KYNEX (gpt-5.4) on ExtractBench. The harness is the local TUI agent: it stages the document, lets the model call tools (``read_page``, ``extract_table``, Document-Intelligence layout analysis, @@ -19,13 +19,13 @@ ``codex_code_extract_*``) this provider runs the extraction as a **local subprocess** -- ``scripts/extract_one.py`` -- so the official environment needs no extra Python packages. The interpreter is taken from -``NIKIT_HARNESS_PYTHON`` (default: the repository's own ``.venv``), and the -repository root from ``NIKIT_HARNESS_ROOT``. +``KYNEX_HARNESS_PYTHON`` (default: the repository's own ``.venv``), and the +repository root from ``KYNEX_HARNESS_ROOT``. To register it with the official CLI, add to ``extract_bench/inference/providers/extract/__init__.py``:: - from .nikit_harness import NikitHarnessExtractProvider # noqa: F401 + from .nikit_harness import KynexExtractProvider # noqa: F401 and to ``extract_bench/inference/pipelines/extract.py``:: @@ -53,7 +53,6 @@ from extract_bench.inference.providers.base import ( Provider, - ProviderConfigError, ProviderPermanentError, ) from extract_bench.schemas.extract_output import ExtractOutput @@ -64,7 +63,7 @@ ) from extract_bench.schemas.product import ProductType -DEFAULT_REPO_ROOT = Path(os.getenv("NIKIT_HARNESS_ROOT") or r"C:\path\to\nikit-harness") +DEFAULT_REPO_ROOT = Path(os.getenv("KYNEX_HARNESS_ROOT") or r"C:\path\to\kynex-harness") # The harness rules that were enabled for the measured 370-document benchmark # run (official ExtractBench scorer). They are passed to the subprocess @@ -84,7 +83,7 @@ def _harness_python() -> str: """The interpreter that has the harness' dependencies.""" - explicit = os.getenv("NIKIT_HARNESS_PYTHON", "").strip() + explicit = os.getenv("KYNEX_HARNESS_PYTHON", "").strip() if explicit: return explicit candidate = DEFAULT_REPO_ROOT / ".venv" / "Scripts" / "python.exe" @@ -97,10 +96,10 @@ async def _extract_async(source: Path, schema: dict[str, Any], timeout: float) - """Run the harness extraction as a subprocess and return the JSON.""" def _run() -> dict[str, Any]: - repo = Path(os.getenv("NIKIT_HARNESS_ROOT") or DEFAULT_REPO_ROOT) + repo = Path(os.getenv("KYNEX_HARNESS_ROOT") or DEFAULT_REPO_ROOT) script = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" if not script.exists(): - raise ProviderPermanentError(f"harness entry point not found: {script} (set NIKIT_HARNESS_ROOT)") + raise ProviderPermanentError(f"harness entry point not found: {script} (set KYNEX_HARNESS_ROOT)") with tempfile.TemporaryDirectory() as td: schema_file = Path(td) / "schema.json" schema_file.write_text(json.dumps(schema), encoding="utf-8") @@ -138,8 +137,8 @@ def _run() -> dict[str, Any]: return await asyncio.get_running_loop().run_in_executor(None, _run) -class NikitHarnessExtractProvider(Provider): - """Provider for the Nikit Harness on ExtractBench. +class KynexExtractProvider(Provider): + """Provider for the KYNEX on ExtractBench. Configuration options: - ``timeout`` (int, default 3600): per-document timeout in seconds. A @@ -154,7 +153,7 @@ def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenceResult: if request.product_type != ProductType.EXTRACT: raise ProviderPermanentError( - f"NikitHarnessExtractProvider only supports EXTRACT product type, got {request.product_type}" + f"KynexExtractProvider only supports EXTRACT product type, got {request.product_type}" ) if not request.schema_override: raise ProviderPermanentError("schema_override is required for EXTRACT product type.") @@ -170,7 +169,7 @@ def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenc timeout=self._timeout, ) ) - except asyncio.TimeoutError as exc: + except TimeoutError as exc: raise ProviderPermanentError(f"harness extraction timed out after {self._timeout}s") from exc except ProviderPermanentError: raise From 7573f4cf7f8c45377a9193217e49b099a073aa65 Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Thu, 22 Oct 2026 00:02:40 +0530 Subject: [PATCH 04/17] Correct the cost figures and state the measurement's limits --- leaderboard.csv | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/leaderboard.csv b/leaderboard.csv index befdf2e1..4923ea01 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -1,4 +1,4 @@ -Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long +Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long LlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2 Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3 Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2 @@ -9,6 +9,7 @@ Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813 Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0 Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6 Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3 +PAEN TUI Harness (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1417,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3 OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1 Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1 @@ -20,34 +21,33 @@ Claude Code (Opus 5.5 Evidence),Coding Agents,86.06,89.63,77.96,80.83,0.1214,0.1 Reducto Extract,Specialized APIs,83.79,92.35,73.29,27.28,0.0457,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0 Extend Extract,Specialized APIs,85.72,89.39,84.14,47.21,0.0680,0.0698,0.0644,0.0625,89.60,85.98,51.76,89.33,84.31,47.29,15.96,21.13,1.03,0.01,53.58,64.50,37.39,0.03,122.3,104.5,170.5 LlamaExtract Agentic,LlamaExtract,95.65,96.06,94.79,94.73,0.0312,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4 -Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3 -Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5 +Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3 +Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5 LlamaExtract Cost-Effective,LlamaExtract,94.21,95.49,91.58,90.86,0.0100,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0 -Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6 -Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7 +Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6 +Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7 GLM-5.3 Flash,Commercial VLM,80.75,96.30,51.56,27.83,0.0034,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9 -Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1 -Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8 -Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9 -Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5 -OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3 -Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7 -NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7 -NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7 -Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2 -Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6 -Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3 -Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6 -GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1 -Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8 -Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2 -Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3 -Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9 -Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 -Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 -Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 -KYNEX (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1429,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 +Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1 +Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8 +Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9 +Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5 +OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3 +Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7 +NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7 +NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7 +Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2 +Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6 +Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3 +Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6 +GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1 +Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8 +Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2 +Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3 +Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9 +Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 +Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 +Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 From b51ca9364402689d00cbef482c559ebda2cea7b8 Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Tue, 20 Oct 2026 17:14:18 -0700 Subject: [PATCH 05/17] Set up pinned KYNEX harness automatically --- README.md | 14 ++ pyproject.toml | 1 + .../inference/pipelines/extract.py | 6 +- .../inference/providers/extract/kynex.py | 177 +++++++++++------- .../inference/providers/extract/test_kynex.py | 141 ++++++++++++++ uv.lock | 2 + 6 files changed, 264 insertions(+), 77 deletions(-) create mode 100644 tests/extract_bench/inference/providers/extract/test_kynex.py diff --git a/README.md b/README.md index 3db7c7e6..4835cb43 100644 --- a/README.md +++ b/README.md @@ -342,6 +342,20 @@ CODEX_API_KEY=... ExtractBench does not use LLM-as-a-judge; all value scoring is deterministic. Your keys only ever call the extraction system you are evaluating. +### KYNEX + +`kynex_gpt_5_4` automatically clones [the harness](https://github.com/Nikit117/kynex-harness) +at revision `4e0d9dcd741173287dc361408143a95f9538de1b` and installs its +`requirements.txt` in an isolated virtual environment on the first extraction. +Git and Python's `venv`/pip support must be available. Subsequent requests reuse +the installation under `~/.cache/extract-bench/kynex/`; set `KYNEX_CACHE_DIR` +to choose another cache location. Concurrent requests share one setup lock. + +Configure `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY`, +`AZURE_OPENAI_CHAT_DEPLOYMENT`, and `AZURE_OPENAI_API_VERSION` for the GPT-5.4 +deployment, plus `AZURE_DI_ENDPOINT` and `AZURE_DI_API_KEY` for Document Intelligence. +Then run `uv run extract-bench run kynex_gpt_5_4 --test`. + ### CLI Reference | Command | Description | diff --git a/pyproject.toml b/pyproject.toml index 68c717a3..58cf60eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "autoevals>=0.0.20", "beautifulsoup4>=4.12.0", "bleach>=6.0.0", + "filelock>=3.13", "fire>=0.7.1", "fuzzysearch>=0.7.3", "huggingface-hub>=0.20.0", diff --git a/src/extract_bench/inference/pipelines/extract.py b/src/extract_bench/inference/pipelines/extract.py index bfb54dc9..70693064 100644 --- a/src/extract_bench/inference/pipelines/extract.py +++ b/src/extract_bench/inference/pipelines/extract.py @@ -951,11 +951,7 @@ def register_extract_pipelines(register_fn) -> None: # type: ignore[no-untyped- ) ) - # KYNEX is a local tool-using agent (gpt-5.4 on Azure OpenAI - # with Azure Document Intelligence for layout). It runs as a subprocess, like - # the other coding-agent pipelines, so the official environment needs no extra - # Python packages: clone https://github.com/Nikit117/kynex-harness and set KYNEX_HARNESS_ROOT to that - # checkout, KYNEX_HARNESS_PYTHON to an interpreter with its dependencies. + # The provider provisions a pinned KYNEX checkout and isolated dependencies. register_fn( _pipeline_spec( pipeline_name="kynex_gpt_5_4", diff --git a/src/extract_bench/inference/providers/extract/kynex.py b/src/extract_bench/inference/providers/extract/kynex.py index 8fcdcc93..5fe3770d 100644 --- a/src/extract_bench/inference/providers/extract/kynex.py +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -1,60 +1,36 @@ -"""Provider for the KYNEX (gpt-5.4) on ExtractBench. - -The harness is the local TUI agent: it stages the document, lets the model call -tools (``read_page``, ``extract_table``, Document-Intelligence layout analysis, -vision) and returns schema-shaped JSON. This provider runs one harness -extraction per request and hands the extracted JSON to the official evaluator. - -It deliberately calls the *same* helpers as the benchmark runner -(``autonomous_document_harness/scripts/run_tui_benchmark_v2.py``), so the -submitted system is the system that was measured: the non-record pruning rules, -the redaction-marker rule, the bulk enumeration of long record lists and the -staging-time OCR fallback for image-only pages all apply. - -The harness returns no per-field citation boxes, so ``field_citations`` is empty -and the grounding metrics are N/A for this system -- the same situation as the -other value-only entries on the leaderboard. - -Like the leaderboard's coding-agent pipelines (``claude_code_extract_*``, -``codex_code_extract_*``) this provider runs the extraction as a **local -subprocess** -- ``scripts/extract_one.py`` -- so the official environment needs -no extra Python packages. The interpreter is taken from -``KYNEX_HARNESS_PYTHON`` (default: the repository's own ``.venv``), and the -repository root from ``KYNEX_HARNESS_ROOT``. - -To register it with the official CLI, add to -``extract_bench/inference/providers/extract/__init__.py``:: - - from .nikit_harness import KynexExtractProvider # noqa: F401 - -and to ``extract_bench/inference/pipelines/extract.py``:: - - register_fn( - _pipeline_spec( - pipeline_name="nikit_harness_gpt_5_4", - provider_name="nikit_harness_extract", - config={"timeout": 3600}, - per_file_timeout=3600.0, - ) - ) +"""Run a pinned KYNEX harness in an automatically provisioned virtual environment. + +The first extraction clones https://github.com/Nikit117/kynex-harness and +installs its requirements into an isolated cache. Later extractions reuse that +installation. Set KYNEX_CACHE_DIR to choose the cache location; Azure OpenAI +and Document Intelligence credentials come from the process environment. + +The upstream agent and post-processing rules run unchanged. No per-field +citation boxes are emitted, so field_citations is empty. """ from __future__ import annotations import asyncio import json +import logging import os import subprocess import sys import tempfile +from collections.abc import Sequence from datetime import datetime from pathlib import Path from typing import Any +from filelock import FileLock, Timeout + from extract_bench.inference.providers.base import ( Provider, + ProviderConfigError, ProviderPermanentError, ) +from extract_bench.inference.providers.registry import register_provider from extract_bench.schemas.extract_output import ExtractOutput from extract_bench.schemas.pipeline_io import ( InferenceRequest, @@ -63,14 +39,13 @@ ) from extract_bench.schemas.product import ProductType -DEFAULT_REPO_ROOT = Path(os.getenv("KYNEX_HARNESS_ROOT") or r"C:\path\to\kynex-harness") +logger = logging.getLogger(__name__) -# The harness rules that were enabled for the measured 370-document benchmark -# run (official ExtractBench scorer). They are passed to the subprocess -# explicitly so the submitted system is the system that was measured: most of -# the rules default to ON in the harness, but ``ADH_BULK_ARRAYS`` is opt-in and -# would otherwise be silently OFF in the official environment, which drops the -# bulk enumeration of long record lists. +HARNESS_REPOSITORY = "https://github.com/Nikit117/kynex-harness.git" +HARNESS_REVISION = "4e0d9dcd741173287dc361408143a95f9538de1b" +_SETUP_TIMEOUT_S = 900 + +# Preserve the rule switches from the submitted benchmark configuration. HARNESS_ENV: dict[str, str] = { "ADH_BULK_ARRAYS": "1", "ADH_PRUNE_NON_RECORDS": "1", @@ -81,31 +56,93 @@ } -def _harness_python() -> str: - """The interpreter that has the harness' dependencies.""" - explicit = os.getenv("KYNEX_HARNESS_PYTHON", "").strip() - if explicit: - return explicit - candidate = DEFAULT_REPO_ROOT / ".venv" / "Scripts" / "python.exe" - if candidate.exists(): - return str(candidate) - return sys.executable +def _run_setup(command: Sequence[str], *, cwd: Path | None = None) -> str: + try: + result = subprocess.run( + command, + cwd=cwd, + check=True, + capture_output=True, + text=True, + timeout=_SETUP_TIMEOUT_S, + ) + except subprocess.CalledProcessError as exc: + detail = exc.stderr if exc.stderr is not None and len(exc.stderr) > 0 else exc.stdout + if detail is None: + detail = str(exc) + raise ProviderConfigError(f"KYNEX setup failed: {detail[-2000:]}") from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise ProviderConfigError(f"KYNEX setup failed: {exc}") from exc + return result.stdout.strip() + + +def _ensure_harness() -> tuple[Path, Path]: + """Publish a complete pinned installation under a process-shared lock.""" + root = Path(os.environ.get("KYNEX_CACHE_DIR", str(Path.home() / ".cache" / "extract-bench" / "kynex"))) + cache = root.expanduser().resolve() / HARNESS_REVISION / f"py{sys.version_info.major}.{sys.version_info.minor}" + cache.mkdir(parents=True, exist_ok=True) + repo = cache / "repo" + environment = cache / "venv" + python = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + ready = cache / "ready" + try: + with FileLock(cache / "setup.lock", timeout=_SETUP_TIMEOUT_S): + if not repo.exists(): + logger.info("Cloning KYNEX at %s", HARNESS_REVISION) + with tempfile.TemporaryDirectory(prefix="clone-", dir=cache) as staging: + checkout = Path(staging) / "repo" + _run_setup( + [ + "git", + "clone", + "--no-checkout", + HARNESS_REPOSITORY, + str(checkout), + ] + ) + _run_setup(["git", "checkout", "--detach", HARNESS_REVISION], cwd=checkout) + checkout.rename(repo) + actual_revision = _run_setup(["git", "rev-parse", "HEAD"], cwd=repo) + changes = _run_setup(["git", "status", "--porcelain"], cwd=repo) + if actual_revision != HARNESS_REVISION or len(changes) > 0: + raise ProviderConfigError(f"Cached KYNEX checkout differs from {HARNESS_REVISION}: {repo}") + if ready.exists() and python.exists() and ready.read_text(encoding="utf-8") == HARNESS_REVISION: + return repo, python + ready.unlink(missing_ok=True) + logger.info("Installing KYNEX dependencies in %s", environment) + _run_setup([sys.executable, "-m", "venv", str(environment)]) + _run_setup( + [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + "--requirement", + str(repo / "requirements.txt"), + ], + cwd=repo, + ) + ready.write_text(HARNESS_REVISION, encoding="utf-8") + except Timeout as exc: + raise ProviderConfigError(f"Timed out waiting for KYNEX setup: {cache}") from exc + return repo, python -async def _extract_async(source: Path, schema: dict[str, Any], timeout: float) -> dict[str, Any]: - """Run the harness extraction as a subprocess and return the JSON.""" +async def _extract_async( + source: Path, schema: dict[str, Any], timeout: float, repo: Path, python: Path +) -> dict[str, Any]: + """Run one extraction in the managed harness environment.""" def _run() -> dict[str, Any]: - repo = Path(os.getenv("KYNEX_HARNESS_ROOT") or DEFAULT_REPO_ROOT) script = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" - if not script.exists(): - raise ProviderPermanentError(f"harness entry point not found: {script} (set KYNEX_HARNESS_ROOT)") with tempfile.TemporaryDirectory() as td: schema_file = Path(td) / "schema.json" schema_file.write_text(json.dumps(schema), encoding="utf-8") out_file = Path(td) / "extraction.json" cmd = [ - _harness_python(), + str(python), "-I", str(script), "--source", @@ -132,19 +169,14 @@ def _run() -> dict[str, Any]: if not out_file.exists(): raise ProviderPermanentError("harness produced no extraction file") extraction = json.loads(out_file.read_text(encoding="utf-8")) - return {"extraction": extraction} + return {"extraction": extraction, "harness_revision": HARNESS_REVISION} return await asyncio.get_running_loop().run_in_executor(None, _run) +@register_provider("kynex_extract") class KynexExtractProvider(Provider): - """Provider for the KYNEX on ExtractBench. - - Configuration options: - - ``timeout`` (int, default 3600): per-document timeout in seconds. A - long document (192 pages) needs the bulk page-wise enumeration, which - is the slowest path in the harness. - """ + """KYNEX extraction with a per-document timeout (default: 3600 seconds).""" def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None): super().__init__(provider_name, base_config) @@ -157,15 +189,16 @@ def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenc ) if not request.schema_override: raise ProviderPermanentError("schema_override is required for EXTRACT product type.") - source = Path(request.source_file_path) + source = Path(request.source_file_path).resolve() if not source.exists(): raise ProviderPermanentError(f"File not found: {source}") + repo, python = _ensure_harness() started_at = datetime.now() try: raw_output = asyncio.run( asyncio.wait_for( - _extract_async(source, request.schema_override, self._timeout), + _extract_async(source, request.schema_override, self._timeout, repo, python), timeout=self._timeout, ) ) @@ -196,7 +229,7 @@ def normalize(self, raw_result: RawInferenceResult) -> InferenceResult: example_id=raw_result.request.example_id, pipeline_name=raw_result.pipeline_name, extracted_data=extracted_data, - field_citations=[], # the harness emits no per-field citation boxes + field_citations=[], ) return InferenceResult( request=raw_result.request, diff --git a/tests/extract_bench/inference/providers/extract/test_kynex.py b/tests/extract_bench/inference/providers/extract/test_kynex.py new file mode 100644 index 00000000..fda3fddc --- /dev/null +++ b/tests/extract_bench/inference/providers/extract/test_kynex.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import subprocess +import sys +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from zipfile import ZipFile + +import pytest + +from extract_bench.inference.pipelines import get_pipeline +from extract_bench.inference.providers.base import ProviderConfigError +from extract_bench.inference.providers.extract import kynex +from extract_bench.inference.providers.registry import create_provider +from extract_bench.schemas.pipeline_io import InferenceRequest +from extract_bench.schemas.product import ProductType + + +def _git(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + +@pytest.fixture +def harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, str]: + """A local harness with a real installable dependency and a newer, wrong HEAD.""" + repo = tmp_path / "upstream" + repo.mkdir() + _git(repo, "init") + _git(repo, "config", "user.email", "fixture@example.invalid") + _git(repo, "config", "user.name", "Test fixture") + wheel = repo / "fixture_dependency-1.0-py3-none-any.whl" + with ZipFile(wheel, "w") as archive: + files = { + "fixture_dependency/__init__.py": 'VALUE = "installed dependency"\n', + "fixture_dependency-1.0.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: fixture-dependency\nVersion: 1.0\n" + ), + "fixture_dependency-1.0.dist-info/WHEEL": ( + "Wheel-Version: 1.0\nGenerator: fixture\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + ), + } + for name, text in files.items(): + archive.writestr(name, text) + record = "fixture_dependency-1.0.dist-info/RECORD" + archive.writestr(record, "".join(f"{name},,\n" for name in [*files, record])) + (repo / "requirements.txt").write_text(f"./{wheel.name}\n", encoding="utf-8") + entry = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" + entry.parent.mkdir(parents=True) + entry.write_text( + "import argparse, json\n" + "from pathlib import Path\n" + "from fixture_dependency import VALUE\n" + "p = argparse.ArgumentParser()\n" + "for name in ('source', 'schema', 'out'): p.add_argument('--' + name, required=True)\n" + "a = p.parse_args()\n" + "assert json.loads(Path(a.schema).read_text())['type'] == 'object'\n" + "Path(a.out).write_text(json.dumps({'value': VALUE, 'source': Path(a.source).read_text()}))\n", + encoding="utf-8", + ) + _git(repo, "add", ".") + _git(repo, "commit", "-m", "Pinned harness") + revision = _git(repo, "rev-parse", "HEAD") + entry.write_text("raise RuntimeError('wrong revision')\n", encoding="utf-8") + _git(repo, "commit", "-am", "Newer upstream revision must not run") + monkeypatch.setattr(kynex, "HARNESS_REPOSITORY", str(repo)) + monkeypatch.setattr(kynex, "HARNESS_REVISION", revision) + monkeypatch.setenv("KYNEX_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setenv("PIP_NO_INDEX", "1") + return repo, revision + + +def test_registered_pipeline_installs_pinned_harness_once_for_concurrent_documents( + harness: tuple[Path, str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Catch missing registration, running upstream HEAD, missing deps, and setup races.""" + pipeline = get_pipeline("kynex_gpt_5_4") + provider = create_provider(pipeline) + setup_commands: list[Sequence[str]] = [] + original = kynex._run_setup + + def track(command: Sequence[str], *, cwd: Path | None = None) -> str: + setup_commands.append(command) + return original(command, cwd=cwd) + + monkeypatch.setattr(kynex, "_run_setup", track) + with ThreadPoolExecutor(max_workers=2) as pool: + installations = list(pool.map(lambda _: kynex._ensure_harness(), range(2))) + repo, python = installations[0] + assert installations[1] == (repo, python) + assert _git(repo, "rev-parse", "HEAD") == harness[1] + assert python != Path(sys.executable) + monkeypatch.chdir(tmp_path) + (tmp_path / "invoice.pdf").write_text("document input", encoding="utf-8") + request = InferenceRequest( + example_id="fixture", + source_file_path="invoice.pdf", + product_type=ProductType.EXTRACT, + schema_override={"type": "object", "properties": {"value": {"type": "string"}}}, + ) + raw = provider.run_inference(pipeline, request) + output = provider.normalize(raw) + assert output.output.extracted_data == { + "value": "installed dependency", + "source": "document input", + } + assert raw.raw_output["harness_revision"] == harness[1] + assert sum("clone" in command for command in setup_commands) == 1 + assert sum("pip" in command for command in setup_commands) == 1 + + (repo / "requirements.txt").write_text("unexpected edit\n", encoding="utf-8") + with pytest.raises(ProviderConfigError, match="differs from"): + kynex._ensure_harness() + + +def test_failed_dependency_install_is_retried_instead_of_reused_as_ready( + harness: tuple[Path, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """An interrupted/failed installation must not publish a reusable environment.""" + original = kynex._run_setup + + def fail_install(command: Sequence[str], *, cwd: Path | None = None) -> str: + if "pip" in command: + raise ProviderConfigError("dependency install interrupted") + return original(command, cwd=cwd) + + monkeypatch.setattr(kynex, "_run_setup", fail_install) + with pytest.raises(ProviderConfigError, match="dependency install interrupted"): + kynex._ensure_harness() + monkeypatch.setattr(kynex, "_run_setup", original) + repo, python = kynex._ensure_harness() + assert _git(repo, "rev-parse", "HEAD") == harness[1] + result = subprocess.check_output( + [ + str(python), + "-c", + "import fixture_dependency; print(fixture_dependency.VALUE)", + ], + text=True, + ) + assert result.strip() == "installed dependency" diff --git a/uv.lock b/uv.lock index d04997e2..7b37147c 100644 --- a/uv.lock +++ b/uv.lock @@ -1617,6 +1617,7 @@ dependencies = [ { name = "autoevals" }, { name = "beautifulsoup4" }, { name = "bleach" }, + { name = "filelock" }, { name = "fire" }, { name = "fuzzysearch" }, { name = "huggingface-hub" }, @@ -1756,6 +1757,7 @@ requires-dist = [ { name = "docling-core", marker = "extra == 'runners'", specifier = ">=2.71.0" }, { name = "extend-ai", marker = "extra == 'extend'", specifier = ">=1.16.0" }, { name = "extend-ai", marker = "extra == 'runners'", specifier = ">=1.16.0" }, + { name = "filelock", specifier = ">=3.13" }, { name = "fire", specifier = ">=0.7.1" }, { name = "fuzzysearch", specifier = ">=0.7.3" }, { name = "google-cloud-documentai", marker = "extra == 'runners'", specifier = ">=2.20.0" }, From f3c220c66b1218110657d7b7b6738d76c52f664d Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Thu, 22 Oct 2026 00:05:08 +0530 Subject: [PATCH 06/17] Correct the leaderboard row: the provider name and the cost cells --- leaderboard.csv | 54 +------------------------------------------------ 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/leaderboard.csv b/leaderboard.csv index 4923ea01..ca4bda9b 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -1,53 +1 @@ -Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long -LlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2 -Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3 -Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2 -Claude Code (Sonnet 5.5 Evidence),Coding Agents,94.68,96.22,92.97,83.69,0.0569,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0 -Codex (GPT-6 Sol Evidence),Coding Agents,94.57,95.99,92.42,87.25,0.0980,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4 -Codex (GPT-6.1 Terra Evidence),Coding Agents,94.28,96.26,91.86,81.21,0.0468,0.0612,0.0179,0.0052,97.23,92.18,95.51,95.86,91.71,81.37,85.69,88.13,81.32,77.21,87.42,88.49,86.10,81.15,86.0,77.3,86.5 -Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2 -Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0 -Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6 -Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3 -PAEN TUI Harness (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1417,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 -Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3 -OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1 -Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1 -Codex (GPT-6 Luna Evidence),Coding Agents,91.01,93.09,87.71,80.88,0.0097,0.0126,0.0040,0.0014,93.17,88.63,85.85,93.07,87.48,80.93,65.70,62.91,71.11,74.36,83.93,84.41,83.47,80.17,206.1,228.9,341.9 -Reducto Deep Extract,Specialized APIs,96.08,96.84,94.80,92.85,0.0635,0.0672,0.0575,0.0469,96.74,95.17,92.28,96.99,94.45,93.45,50.76,53.49,46.08,40.19,69.75,72.92,62.27,68.96,134.1,182.3,511.1 -DeepSeek V4.1 Flash,Commercial VLM,87.11,94.44,81.49,22.23,0.0032,0.0037,0.0017,0.0006,94.06,81.59,22.16,94.98,81.73,22.31,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,9.9,29.2,29.0 -Claude Code (Opus 4.8 Evidence),Coding Agents,86.84,90.54,79.17,77.94,0.6324,0.8721,0.2802,0.0959,90.69,79.27,81.97,90.40,79.08,77.48,49.48,45.31,62.35,58.43,72.52,74.84,67.38,68.03,274.0,347.1,501.9 -Claude Code (Opus 5.5 Evidence),Coding Agents,86.06,89.63,77.96,80.83,0.1214,0.1659,0.0530,0.0192,89.62,77.77,80.60,89.69,78.34,81.06,71.55,73.57,65.54,74.87,76.95,79.27,71.36,76.47,73.6,95.6,227.9 -Reducto Extract,Specialized APIs,83.79,92.35,73.29,27.28,0.0457,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0 -Extend Extract,Specialized APIs,85.72,89.39,84.14,47.21,0.0680,0.0698,0.0644,0.0625,89.60,85.98,51.76,89.33,84.31,47.29,15.96,21.13,1.03,0.01,53.58,64.50,37.39,0.03,122.3,104.5,170.5 -LlamaExtract Agentic,LlamaExtract,95.65,96.06,94.79,94.73,0.0312,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4 -Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3 -Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5 -LlamaExtract Cost-Effective,LlamaExtract,94.21,95.49,91.58,90.86,0.0100,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0 -Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6 -Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7 -GLM-5.3 Flash,Commercial VLM,80.75,96.30,51.56,27.83,0.0034,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9 -Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1 -Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8 -Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9 -Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5 -OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3 -Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7 -NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7 -NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7 -Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2 -Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6 -Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3 -Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6 -GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1 -Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8 -Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2 -Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3 -Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9 -Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 -Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 -Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 +Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_LongLlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2Claude Code (Sonnet 5.5 Evidence),Coding Agents,94.68,96.22,92.97,83.69,0.0569,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0Codex (GPT-6 Sol Evidence),Coding Agents,94.57,95.99,92.42,87.25,0.0980,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4Codex (GPT-6.1 Terra Evidence),Coding Agents,94.28,96.26,91.86,81.21,0.0468,0.0612,0.0179,0.0052,97.23,92.18,95.51,95.86,91.71,81.37,85.69,88.13,81.32,77.21,87.42,88.49,86.10,81.15,86.0,77.3,86.5Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3KYNEX (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1417,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1Codex (GPT-6 Luna Evidence),Coding Agents,91.01,93.09,87.71,80.88,0.0097,0.0126,0.0040,0.0014,93.17,88.63,85.85,93.07,87.48,80.93,65.70,62.91,71.11,74.36,83.93,84.41,83.47,80.17,206.1,228.9,341.9Reducto Deep Extract,Specialized APIs,96.08,96.84,94.80,92.85,0.0635,0.0672,0.0575,0.0469,96.74,95.17,92.28,96.99,94.45,93.45,50.76,53.49,46.08,40.19,69.75,72.92,62.27,68.96,134.1,182.3,511.1DeepSeek V4.1 Flash,Commercial VLM,87.11,94.44,81.49,22.23,0.0032,0.0037,0.0017,0.0006,94.06,81.59,22.16,94.98,81.73,22.31,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,9.9,29.2,29.0Claude Code (Opus 4.8 Evidence),Coding Agents,86.84,90.54,79.17,77.94,0.6324,0.8721,0.2802,0.0959,90.69,79.27,81.97,90.40,79.08,77.48,49.48,45.31,62.35,58.43,72.52,74.84,67.38,68.03,274.0,347.1,501.9Claude Code (Opus 5.5 Evidence),Coding Agents,86.06,89.63,77.96,80.83,0.1214,0.1659,0.0530,0.0192,89.62,77.77,80.60,89.69,78.34,81.06,71.55,73.57,65.54,74.87,76.95,79.27,71.36,76.47,73.6,95.6,227.9Reducto Extract,Specialized APIs,83.79,92.35,73.29,27.28,0.0457,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0Extend Extract,Specialized APIs,85.72,89.39,84.14,47.21,0.0680,0.0698,0.0644,0.0625,89.60,85.98,51.76,89.33,84.31,47.29,15.96,21.13,1.03,0.01,53.58,64.50,37.39,0.03,122.3,104.5,170.5LlamaExtract Agentic,LlamaExtract,95.65,96.06,94.79,94.73,0.0312,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5LlamaExtract Cost-Effective,LlamaExtract,94.21,95.49,91.58,90.86,0.0100,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7GLM-5.3 Flash,Commercial VLM,80.75,96.30,51.56,27.83,0.0034,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 \ No newline at end of file From 6d607a5fddd3dfa48f806b771f5d9c80c6d97324 Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Thu, 22 Oct 2026 00:18:11 +0530 Subject: [PATCH 07/17] Pin the harness to the current tip of the published repository The pinned revision now includes the openai-compatible transport contributed in Nikit117/kynex-harness#1, so the pipeline runs the same code that is in the repository. --- src/extract_bench/inference/providers/extract/kynex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract_bench/inference/providers/extract/kynex.py b/src/extract_bench/inference/providers/extract/kynex.py index 5fe3770d..a3b6f978 100644 --- a/src/extract_bench/inference/providers/extract/kynex.py +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) HARNESS_REPOSITORY = "https://github.com/Nikit117/kynex-harness.git" -HARNESS_REVISION = "4e0d9dcd741173287dc361408143a95f9538de1b" +HARNESS_REVISION = "d50a78d9ee99a076e05e8d8942dfe3e2152e9065" _SETUP_TIMEOUT_S = 900 # Preserve the rule switches from the submitted benchmark configuration. From 6740d1cd74cb2456dd1101aa0d9a0624feb9080c Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Wed, 28 Oct 2026 01:53:27 +0530 Subject: [PATCH 08/17] Fix the leaderboard CSV and regenerate the README tables The rebase corrupted the CSV (one line, no newlines) and left the README tables without the KYNEX row. This restores leaderboard.csv from main's rows plus the corrected KYNEX entry (92.65 overall, 0.1417 USD per page), and regenerates all three README tables with scripts/update_readme.py so the --check passes. --- README.md | 58 ++++++++++++++++++++++++------------------------- leaderboard.csv | 54 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 4835cb43..e7254893 100644 --- a/README.md +++ b/README.md @@ -49,26 +49,26 @@ In the official benchmark, documents that fail or exceed a 30-minute limit score ### ExtractBench leaderboard -**Unified value F1** — rows are ordered by Overall Value F1, then lower cost. +**Unified value F1** — rows are ordered by Overall Value F1, then lower cost. -| Rank | Provider | Category | Overall | Short | Medium | Long | ¢ / Page | +| Rank | Provider | Category | Overall | Short | Medium | Long | ¢ / Page | |---:|---|---|---:|---:|---:|---:|---:| -| 1 | LlamaExtract Agentic Plus | LlamaExtract | **96.17** | 97.02 | 94.30 | 94.65 | 7.50¢ | -| 2 | Reducto Deep Extract | Specialized APIs | 96.08 | 96.84 | **94.80** | 92.85 | 6.35¢ | -| 3 | Pulse (Effort) | Specialized APIs | 96.07 | 96.83 | 94.70 | 93.24 | 10.50¢ | -| 4 | LlamaExtract Agentic | LlamaExtract | 95.65 | 96.06 | 94.79 | **94.73** | 3.12¢ | -| 5 | Codex (GPT-6.1 Sol Evidence) | Coding Agents | 95.31 | 97.01 | 92.96 | 85.46 | 7.13¢ | -| 6 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ | -| 7 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ | -| 8 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ | -| 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ | -| 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ | - -Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv). +| 1 | LlamaExtract Agentic Plus | LlamaExtract | **96.17** | 97.02 | 94.30 | 94.65 | 7.50¢ | +| 2 | Reducto Deep Extract | Specialized APIs | 96.08 | 96.84 | **94.80** | 92.85 | 6.35¢ | +| 3 | Pulse (Effort) | Specialized APIs | 96.07 | 96.83 | 94.70 | 93.24 | 10.50¢ | +| 4 | LlamaExtract Agentic | LlamaExtract | 95.65 | 96.06 | 94.79 | **94.73** | 3.12¢ | +| 5 | Codex (GPT-6.1 Sol Evidence) | Coding Agents | 95.31 | 97.01 | 92.96 | 85.46 | 7.13¢ | +| 6 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ | +| 7 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ | +| 8 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ | +| 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ | +| 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ | + +Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). -**Grounding F1** — rows are ordered by overall word-level grounding F1. +**Grounding F1** — rows are ordered by overall word-level grounding F1. @@ -89,7 +89,7 @@ Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv).
-Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv). +Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). ### ExtractBench Pro leaderboard @@ -97,20 +97,20 @@ Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv). Rows are ordered by % Perfect, then Value F1, then lower cost. -| Rank | Provider | Category | % Perfect | Value F1 | Grounding F1 | ¢ / Page | +| Rank | Provider | Category | % Perfect | Value F1 | Grounding F1 | ¢ / Page | |---:|---|---|---:|---:|---:|---:| -| 1 | Codex GPT-6 Astra | Coding Agents | **58** | **94.53** | **70.97** | 95.20¢ | -| 2 | Claude Code Opus 5.5 | Coding Agents | 51 | 74.87 | 45.29 | 41.20¢ | -| 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | 77.70 | 60.39 | 7.50¢ | -| 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ | -| 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ | -| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.98 | 33.90 | 36.16¢ | -| 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ | -| 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ | -| 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | 63.96 | 13.86¢ | -| 10 | Reducto Deep | Specialized APIs | 3 | 56.35 | 42.20 | 6.18¢ | - -Top 10 of 13 systems — full table in [leaderboard-pro.csv](leaderboard-pro.csv). Scores with no time limit are also available there. +| 1 | Codex GPT-6 Astra | Coding Agents | **58** | **94.53** | **70.97** | 95.20¢ | +| 2 | Claude Code Opus 5.5 | Coding Agents | 51 | 74.87 | 45.29 | 41.20¢ | +| 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | 77.70 | 60.39 | 7.50¢ | +| 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ | +| 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ | +| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.98 | 33.90 | 36.16¢ | +| 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ | +| 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ | +| 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | 63.96 | 13.86¢ | +| 10 | Reducto Deep | Specialized APIs | 3 | 56.35 | 42.20 | 6.18¢ | + +Top 10 of 13 systems — full table in [leaderboard-pro.csv](leaderboard-pro.csv). Scores with no time limit are also available there. ### Submitting results diff --git a/leaderboard.csv b/leaderboard.csv index ca4bda9b..91c407d5 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -1 +1,53 @@ -Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_LongLlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2Claude Code (Sonnet 5.5 Evidence),Coding Agents,94.68,96.22,92.97,83.69,0.0569,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0Codex (GPT-6 Sol Evidence),Coding Agents,94.57,95.99,92.42,87.25,0.0980,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4Codex (GPT-6.1 Terra Evidence),Coding Agents,94.28,96.26,91.86,81.21,0.0468,0.0612,0.0179,0.0052,97.23,92.18,95.51,95.86,91.71,81.37,85.69,88.13,81.32,77.21,87.42,88.49,86.10,81.15,86.0,77.3,86.5Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3KYNEX (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1417,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1Codex (GPT-6 Luna Evidence),Coding Agents,91.01,93.09,87.71,80.88,0.0097,0.0126,0.0040,0.0014,93.17,88.63,85.85,93.07,87.48,80.93,65.70,62.91,71.11,74.36,83.93,84.41,83.47,80.17,206.1,228.9,341.9Reducto Deep Extract,Specialized APIs,96.08,96.84,94.80,92.85,0.0635,0.0672,0.0575,0.0469,96.74,95.17,92.28,96.99,94.45,93.45,50.76,53.49,46.08,40.19,69.75,72.92,62.27,68.96,134.1,182.3,511.1DeepSeek V4.1 Flash,Commercial VLM,87.11,94.44,81.49,22.23,0.0032,0.0037,0.0017,0.0006,94.06,81.59,22.16,94.98,81.73,22.31,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,9.9,29.2,29.0Claude Code (Opus 4.8 Evidence),Coding Agents,86.84,90.54,79.17,77.94,0.6324,0.8721,0.2802,0.0959,90.69,79.27,81.97,90.40,79.08,77.48,49.48,45.31,62.35,58.43,72.52,74.84,67.38,68.03,274.0,347.1,501.9Claude Code (Opus 5.5 Evidence),Coding Agents,86.06,89.63,77.96,80.83,0.1214,0.1659,0.0530,0.0192,89.62,77.77,80.60,89.69,78.34,81.06,71.55,73.57,65.54,74.87,76.95,79.27,71.36,76.47,73.6,95.6,227.9Reducto Extract,Specialized APIs,83.79,92.35,73.29,27.28,0.0457,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0Extend Extract,Specialized APIs,85.72,89.39,84.14,47.21,0.0680,0.0698,0.0644,0.0625,89.60,85.98,51.76,89.33,84.31,47.29,15.96,21.13,1.03,0.01,53.58,64.50,37.39,0.03,122.3,104.5,170.5LlamaExtract Agentic,LlamaExtract,95.65,96.06,94.79,94.73,0.0312,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5LlamaExtract Cost-Effective,LlamaExtract,94.21,95.49,91.58,90.86,0.0100,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7GLM-5.3 Flash,Commercial VLM,80.75,96.30,51.56,27.83,0.0034,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 \ No newline at end of file +Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long +LlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2 +Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3 +Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2 +Claude Code (Sonnet 5.5 Evidence),Coding Agents,94.68,96.22,92.97,83.69,0.0569,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0 +Codex (GPT-6 Sol Evidence),Coding Agents,94.57,95.99,92.42,87.25,0.0980,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4 +Codex (GPT-6.1 Terra Evidence),Coding Agents,94.28,96.26,91.86,81.21,0.0468,0.0612,0.0179,0.0052,97.23,92.18,95.51,95.86,91.71,81.37,85.69,88.13,81.32,77.21,87.42,88.49,86.10,81.15,86.0,77.3,86.5 +Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2 +Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0 +Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6 +Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3 +KYNEX (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1417,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 +Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3 +OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1 +Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1 +Codex (GPT-6 Luna Evidence),Coding Agents,91.01,93.09,87.71,80.88,0.0097,0.0126,0.0040,0.0014,93.17,88.63,85.85,93.07,87.48,80.93,65.70,62.91,71.11,74.36,83.93,84.41,83.47,80.17,206.1,228.9,341.9 +Reducto Deep Extract,Specialized APIs,96.08,96.84,94.80,92.85,0.0635,0.0672,0.0575,0.0469,96.74,95.17,92.28,96.99,94.45,93.45,50.76,53.49,46.08,40.19,69.75,72.92,62.27,68.96,134.1,182.3,511.1 +DeepSeek V4.1 Flash,Commercial VLM,87.11,94.44,81.49,22.23,0.0032,0.0037,0.0017,0.0006,94.06,81.59,22.16,94.98,81.73,22.31,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,9.9,29.2,29.0 +Claude Code (Opus 4.8 Evidence),Coding Agents,86.84,90.54,79.17,77.94,0.6324,0.8721,0.2802,0.0959,90.69,79.27,81.97,90.40,79.08,77.48,49.48,45.31,62.35,58.43,72.52,74.84,67.38,68.03,274.0,347.1,501.9 +Claude Code (Opus 5.5 Evidence),Coding Agents,86.06,89.63,77.96,80.83,0.1214,0.1659,0.0530,0.0192,89.62,77.77,80.60,89.69,78.34,81.06,71.55,73.57,65.54,74.87,76.95,79.27,71.36,76.47,73.6,95.6,227.9 +Reducto Extract,Specialized APIs,83.79,92.35,73.29,27.28,0.0457,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0 +Extend Extract,Specialized APIs,85.72,89.39,84.14,47.21,0.0680,0.0698,0.0644,0.0625,89.60,85.98,51.76,89.33,84.31,47.29,15.96,21.13,1.03,0.01,53.58,64.50,37.39,0.03,122.3,104.5,170.5 +LlamaExtract Agentic,LlamaExtract,95.65,96.06,94.79,94.73,0.0312,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4 +Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3 +Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5 +LlamaExtract Cost-Effective,LlamaExtract,94.21,95.49,91.58,90.86,0.0100,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0 +Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6 +Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7 +GLM-5.3 Flash,Commercial VLM,80.75,96.30,51.56,27.83,0.0034,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9 +Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1 +Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8 +Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9 +Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5 +OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3 +Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7 +NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7 +NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7 +Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2 +Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6 +Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3 +Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6 +GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1 +Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8 +Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2 +Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3 +Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9 +Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 +Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 +Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 From 7e0784467069ad2db4925d0cccd5c6378b866adb Mon Sep 17 00:00:00 2001 From: Nikit Kashyap Date: Wed, 28 Oct 2026 02:09:52 +0530 Subject: [PATCH 09/17] Fix UTF-8 encoding in README.md (CI build failure) Replace Windows-1252 bytes (0x97 em dash, 0xa2 cent sign) with proper UTF-8 sequences so hatchling can read the readme during uv sync/build. Co-Authored-By: Claude Opus 4.6 --- README.md | 58 +++++++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index e7254893..9c5d830f 100644 --- a/README.md +++ b/README.md @@ -49,26 +49,26 @@ In the official benchmark, documents that fail or exceed a 30-minute limit score ### ExtractBench leaderboard -**Unified value F1** — rows are ordered by Overall Value F1, then lower cost. +**Unified value F1** — rows are ordered by Overall Value F1, then lower cost. -| Rank | Provider | Category | Overall | Short | Medium | Long | ¢ / Page | +| Rank | Provider | Category | Overall | Short | Medium | Long | ¢ / Page | |---:|---|---|---:|---:|---:|---:|---:| -| 1 | LlamaExtract Agentic Plus | LlamaExtract | **96.17** | 97.02 | 94.30 | 94.65 | 7.50¢ | -| 2 | Reducto Deep Extract | Specialized APIs | 96.08 | 96.84 | **94.80** | 92.85 | 6.35¢ | -| 3 | Pulse (Effort) | Specialized APIs | 96.07 | 96.83 | 94.70 | 93.24 | 10.50¢ | -| 4 | LlamaExtract Agentic | LlamaExtract | 95.65 | 96.06 | 94.79 | **94.73** | 3.12¢ | -| 5 | Codex (GPT-6.1 Sol Evidence) | Coding Agents | 95.31 | 97.01 | 92.96 | 85.46 | 7.13¢ | -| 6 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ | -| 7 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ | -| 8 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ | -| 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ | -| 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ | - -Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). +| 1 | LlamaExtract Agentic Plus | LlamaExtract | **96.17** | 97.02 | 94.30 | 94.65 | 7.50¢ | +| 2 | Reducto Deep Extract | Specialized APIs | 96.08 | 96.84 | **94.80** | 92.85 | 6.35¢ | +| 3 | Pulse (Effort) | Specialized APIs | 96.07 | 96.83 | 94.70 | 93.24 | 10.50¢ | +| 4 | LlamaExtract Agentic | LlamaExtract | 95.65 | 96.06 | 94.79 | **94.73** | 3.12¢ | +| 5 | Codex (GPT-6.1 Sol Evidence) | Coding Agents | 95.31 | 97.01 | 92.96 | 85.46 | 7.13¢ | +| 6 | Claude Code (Sonnet 5.5 Evidence) | Coding Agents | 94.68 | 96.22 | 92.97 | 83.69 | 5.69¢ | +| 7 | Codex (GPT-6 Sol Evidence) | Coding Agents | 94.57 | 95.99 | 92.42 | 87.25 | 9.80¢ | +| 8 | Pulse (Non-effort) | Specialized APIs | 94.51 | 95.39 | 92.49 | 93.31 | 3.00¢ | +| 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ | +| 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ | + +Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). -**Grounding F1** — rows are ordered by overall word-level grounding F1. +**Grounding F1** — rows are ordered by overall word-level grounding F1. @@ -89,7 +89,7 @@ Top 10 of 52 systems
-Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). +Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). ### ExtractBench Pro leaderboard @@ -97,20 +97,20 @@ Top 10 of 52 systems Rows are ordered by % Perfect, then Value F1, then lower cost. -| Rank | Provider | Category | % Perfect | Value F1 | Grounding F1 | ¢ / Page | +| Rank | Provider | Category | % Perfect | Value F1 | Grounding F1 | ¢ / Page | |---:|---|---|---:|---:|---:|---:| -| 1 | Codex GPT-6 Astra | Coding Agents | **58** | **94.53** | **70.97** | 95.20¢ | -| 2 | Claude Code Opus 5.5 | Coding Agents | 51 | 74.87 | 45.29 | 41.20¢ | -| 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | 77.70 | 60.39 | 7.50¢ | -| 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ | -| 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ | -| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.98 | 33.90 | 36.16¢ | -| 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ | -| 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ | -| 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | 63.96 | 13.86¢ | -| 10 | Reducto Deep | Specialized APIs | 3 | 56.35 | 42.20 | 6.18¢ | - -Top 10 of 13 systems — full table in [leaderboard-pro.csv](leaderboard-pro.csv). Scores with no time limit are also available there. +| 1 | Codex GPT-6 Astra | Coding Agents | **58** | **94.53** | **70.97** | 95.20¢ | +| 2 | Claude Code Opus 5.5 | Coding Agents | 51 | 74.87 | 45.29 | 41.20¢ | +| 3 | LlamaExtract Agentic Plus | LlamaExtract | 38 | 77.70 | 60.39 | 7.50¢ | +| 4 | Extend Operator | Specialized APIs | 34 | 73.87 | 57.75 | 9.88¢ | +| 5 | Claude Code Sonnet 5.5 | Coding Agents | 32 | 73.62 | 40.47 | 19.98¢ | +| 6 | Gemini CLI (Gemini 4 Argon Medium) | Coding Agents | 30 | 42.98 | 33.90 | 36.16¢ | +| 7 | LlamaExtract Agentic | LlamaExtract | 20 | 71.32 | 58.01 | 3.12¢ | +| 8 | Pulse Effort | Specialized APIs | 18 | 70.53 | 18.74 | 10.50¢ | +| 9 | Codex GPT-6.1 Sol | Coding Agents | 12 | 74.06 | 63.96 | 13.86¢ | +| 10 | Reducto Deep | Specialized APIs | 3 | 56.35 | 42.20 | 6.18¢ | + +Top 10 of 13 systems — full table in [leaderboard-pro.csv](leaderboard-pro.csv). Scores with no time limit are also available there. ### Submitting results From 0271d154f962caa1b8191466f6e809ccfe0f0186 Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Wed, 28 Oct 2026 15:53:03 -0400 Subject: [PATCH 10/17] Update harness pin --- README.md | 2 +- src/extract_bench/inference/providers/extract/kynex.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c5d830f..6efe15bb 100644 --- a/README.md +++ b/README.md @@ -345,7 +345,7 @@ ExtractBench does not use LLM-as-a-judge; all value scoring is deterministic. Yo ### KYNEX `kynex_gpt_5_4` automatically clones [the harness](https://github.com/Nikit117/kynex-harness) -at revision `4e0d9dcd741173287dc361408143a95f9538de1b` and installs its +at revision `87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91` and installs its `requirements.txt` in an isolated virtual environment on the first extraction. Git and Python's `venv`/pip support must be available. Subsequent requests reuse the installation under `~/.cache/extract-bench/kynex/`; set `KYNEX_CACHE_DIR` diff --git a/src/extract_bench/inference/providers/extract/kynex.py b/src/extract_bench/inference/providers/extract/kynex.py index a3b6f978..2bf23e21 100644 --- a/src/extract_bench/inference/providers/extract/kynex.py +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) HARNESS_REPOSITORY = "https://github.com/Nikit117/kynex-harness.git" -HARNESS_REVISION = "d50a78d9ee99a076e05e8d8942dfe3e2152e9065" +HARNESS_REVISION = "87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91" _SETUP_TIMEOUT_S = 900 # Preserve the rule switches from the submitted benchmark configuration. From b17d83958619c426f7cf19b2c7ee9bebfbdbe4dc Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 29 Oct 2026 18:00:30 -0700 Subject: [PATCH 11/17] Update KYNEX results --- README.md | 4 ++++ docs/extractbench.md | 17 +++++++++++++++++ leaderboard.csv | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e627049a..03678151 100644 --- a/README.md +++ b/README.md @@ -356,6 +356,10 @@ Configure `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY`, deployment, plus `AZURE_DI_ENDPOINT` and `AZURE_DI_API_KEY` for Document Intelligence. Then run `uv run extract-bench run kynex_gpt_5_4 --test`. +Independent verification on all 370 documents gives **92.97% overall value F1** +and an estimated **4.96¢ per page**. Split metrics and pricing are documented +in [KYNEX verification](docs/extractbench.md#kynex-verification). + ### CLI Reference | Command | Description | diff --git a/docs/extractbench.md b/docs/extractbench.md index 02d82d1e..ad1ba326 100644 --- a/docs/extractbench.md +++ b/docs/extractbench.md @@ -77,6 +77,23 @@ The four self-hosted pipelines need an endpoint you run yourself; see [.env.exam The three LlamaExtract leaderboard pipelines select Extract v2.6 with citations enabled and no custom Parse configuration. All three return word-level citation boxes. +### KYNEX verification + +[Independent verification](https://github.com/run-llama/ExtractBench/pull/84#issuecomment-6512168305) covers all 370 original documents using GPT-5.4 and Azure Document Intelligence `prebuilt-layout`, with harness revision `87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91`. + +| Split | Documents | Value F1 | Precision | Recall | ¢ / page | Seconds / document | +|---|---:|---:|---:|---:|---:|---:| +| Short | 252 | 94.29 | 94.04 | 94.90 | 8.42 | 118.8 | +| Medium | 98 | 92.48 | 93.55 | 91.75 | 3.46 | 245.3 | +| Long | 20 | 78.77 | 85.61 | 75.46 | 5.80 | 653.9 | +| Overall | 370 | 92.97 | 93.45 | 93.02 | 4.96 | 181.3 | + +Value scores are document-level macro averages from the pinned official scorer. The harness emits no field citations, so word- and page-grounding scores remain zero. Latency is the mean recorded provider duration per document. + +The full-workload estimate is **$241.53** over 4,869 source pages: **$191.64** for model calls and **$49.89** for Document Intelligence. Prices are $2.50 / $0.25 / $15 per million uncached-input / cached-input / output tokens, with the long-context multipliers applied per request, and $10 per 1,000 Layout pages. Free-tier credits are excluded. + +Pricing starts with a cold prompt cache for each document, then assumes hits for the full unchanged input prefix from earlier completed calls within that document, including conversation history and tool results. It uses a 1,024-token minimum and rounds cached prefixes down to 128-token increments. Cross-document sharing is excluded. Costs are estimates from known usage; unmetered failed requests are additional. + ## Cost of a full run Costs use each provider's official listed price. diff --git a/leaderboard.csv b/leaderboard.csv index 91c407d5..85d87f63 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -9,7 +9,7 @@ Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813 Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0 Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6 Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3 -KYNEX (gpt-5.4),Coding Agents,92.65,94.12,91.52,79.73,0.1417,0.6111,0.0979,0.0448,93.42,92.65,84.81,95.07,90.84,77.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,101.9,271.0,867.2 +KYNEX (gpt-5.4),Coding Agents,92.97,94.29,92.48,78.77,0.0496,0.0842,0.0346,0.0580,94.04,93.55,85.61,94.90,91.75,75.46,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,118.8,245.3,653.9 Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3 OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1 Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1 From 15a881ea90c759c42dfe64662e1ec012819cca7e Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 29 Oct 2026 18:02:34 -0700 Subject: [PATCH 12/17] Trim README whitespace --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 03678151..2d602748 100644 --- a/README.md +++ b/README.md @@ -454,4 +454,3 @@ src/extract_bench/ ## License The code and both datasets (ExtractBench and ExtractBench Pro) are released under the [Apache 2.0 License](LICENSE). - From 8e4180f7953b663404633639040614520262776f Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 29 Oct 2026 18:05:34 -0700 Subject: [PATCH 13/17] Remove KYNEX README section --- README.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/README.md b/README.md index 2d602748..8d8b10f8 100644 --- a/README.md +++ b/README.md @@ -342,24 +342,6 @@ CODEX_API_KEY=... ExtractBench does not use LLM-as-a-judge; all value scoring is deterministic. Your keys only ever call the extraction system you are evaluating. -### KYNEX - -`kynex_gpt_5_4` automatically clones [the harness](https://github.com/Nikit117/kynex-harness) -at revision `87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91` and installs its -`requirements.txt` in an isolated virtual environment on the first extraction. -Git and Python's `venv`/pip support must be available. Subsequent requests reuse -the installation under `~/.cache/extract-bench/kynex/`; set `KYNEX_CACHE_DIR` -to choose another cache location. Concurrent requests share one setup lock. - -Configure `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY`, -`AZURE_OPENAI_CHAT_DEPLOYMENT`, and `AZURE_OPENAI_API_VERSION` for the GPT-5.4 -deployment, plus `AZURE_DI_ENDPOINT` and `AZURE_DI_API_KEY` for Document Intelligence. -Then run `uv run extract-bench run kynex_gpt_5_4 --test`. - -Independent verification on all 370 documents gives **92.97% overall value F1** -and an estimated **4.96¢ per page**. Split metrics and pricing are documented -in [KYNEX verification](docs/extractbench.md#kynex-verification). - ### CLI Reference | Command | Description | From c480b31352e54a8fba0cb0b1c34139a9ab1f964f Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 29 Oct 2026 18:07:44 -0700 Subject: [PATCH 14/17] Preserve CSV line endings --- leaderboard.csv | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/leaderboard.csv b/leaderboard.csv index 85d87f63..aaa425d5 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -1,4 +1,4 @@ -Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long +Provider,Category,Overall,Short,Medium,Long,Cost_Per_Page,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long LlamaExtract Agentic Plus,LlamaExtract,96.17,97.02,94.30,94.65,0.0750,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2 Pulse (Effort),Specialized APIs,96.07,96.83,94.70,93.24,0.1050,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3 Codex (GPT-6.1 Sol Evidence),Coding Agents,95.31,97.01,92.96,85.46,0.0713,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2 @@ -21,33 +21,33 @@ Claude Code (Opus 5.5 Evidence),Coding Agents,86.06,89.63,77.96,80.83,0.1214,0.1 Reducto Extract,Specialized APIs,83.79,92.35,73.29,27.28,0.0457,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0 Extend Extract,Specialized APIs,85.72,89.39,84.14,47.21,0.0680,0.0698,0.0644,0.0625,89.60,85.98,51.76,89.33,84.31,47.29,15.96,21.13,1.03,0.01,53.58,64.50,37.39,0.03,122.3,104.5,170.5 LlamaExtract Agentic,LlamaExtract,95.65,96.06,94.79,94.73,0.0312,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4 -Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3 -Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5 +Qwen3.6 35B,OSS,88.11,92.86,86.37,36.90,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3 +Claude Code (Opus 4.8),Coding Agents,87.09,90.08,79.21,88.07,0.1617,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5 LlamaExtract Cost-Effective,LlamaExtract,94.21,95.49,91.58,90.86,0.0100,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0 -Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6 -Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7 +Extend (Max Context),Specialized APIs,88.62,92.32,78.75,90.46,0.1000,0.1000,0.1000,0.1000,92.39,77.36,93.11,92.28,80.84,89.86,25.20,33.93,0.21,0.02,49.04,61.71,27.68,0.03,86.2,130.6,364.6 +Datalab (Accurate + Balanced),Specialized APIs,85.70,89.40,85.08,42.04,0.0350,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7 GLM-5.3 Flash,Commercial VLM,80.75,96.30,51.56,27.83,0.0034,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9 -Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1 -Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8 -Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9 -Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5 -OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3 -Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7 -NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7 -NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, -Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7 -Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2 -Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6 -Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3 -Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6 -GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1 -Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8 -Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2 -Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3 -Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9 -Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 -Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 -Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 +Google Gemini 3.8 Flash,Commercial VLM,80.71,88.14,72.23,28.70,0.0043,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1 +Google Gemini 3.5 Flash,Commercial VLM,79.84,87.87,69.76,27.90,0.0100,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8 +Reducto Extract (v4),Specialized APIs,79.58,92.20,59.45,19.29,0.0459,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9 +Lift Datalab 9B,OSS,78.43,87.64,65.69,24.78,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5 +OpenAI GPT-5.4 Nano,Commercial VLM,74.90,77.43,76.37,35.81,0.0021,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3 +Gemma4 26B,OSS,70.27,77.70,59.61,28.88,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7 +NuExtract3,OSS,81.02,86.41,77.01,32.71,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7 +NuExtract 2.0 8B,OSS,49.42,62.92,23.50,6.39,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +InternVL3.5 8B,OSS,64.63,79.26,36.62,17.57,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +InternVL3.5 14B,OSS,66.35,80.70,39.04,19.41,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +MiniCPM-V 4.5,OSS,45.85,57.39,25.30,1.13,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,, +Qwen3.5 35B,OSS,88.00,93.52,85.27,31.73,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7 +Qwen3.5 9B,OSS,86.10,92.31,81.12,32.22,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2 +Gemma4 E4B,OSS,69.64,81.10,48.93,26.82,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6 +Gemma4 E2B,OSS,51.85,63.71,28.96,14.58,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3 +Kimi-VL A3B,OSS,57.77,69.86,34.61,18.94,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6 +GLM-4.6V Flash,OSS,45.19,47.28,44.52,22.17,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1 +Qwen3.8 Flash Next,OSS,89.88,94.82,87.81,37.74,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8 +Qwen3.8 27B,OSS,89.75,94.68,87.54,38.45,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2 +Qwen3.5 2B,OSS,63.40,71.22,50.93,25.93,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3 +Qwen3.5 0.8B,OSS,46.63,57.44,26.98,6.65,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9 +Gemma4 12B,OSS,45.39,53.62,30.82,13.16,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4 +Qwen3.5 4B,OSS,82.43,89.06,76.64,27.31,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6 +Kimi K3,Commercial VLM,83.17,94.64,69.64,4.99,0.0675,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4 From b6081ce52c9433eb844728a51b00f7e920b2896e Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Thu, 29 Oct 2026 18:13:35 -0700 Subject: [PATCH 15/17] Limit KYNEX runs to 30 minutes --- docs/extractbench.md | 17 ----------------- .../inference/pipelines/extract.py | 6 ++---- .../inference/providers/extract/kynex.py | 4 ++-- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/docs/extractbench.md b/docs/extractbench.md index ad1ba326..02d82d1e 100644 --- a/docs/extractbench.md +++ b/docs/extractbench.md @@ -77,23 +77,6 @@ The four self-hosted pipelines need an endpoint you run yourself; see [.env.exam The three LlamaExtract leaderboard pipelines select Extract v2.6 with citations enabled and no custom Parse configuration. All three return word-level citation boxes. -### KYNEX verification - -[Independent verification](https://github.com/run-llama/ExtractBench/pull/84#issuecomment-6512168305) covers all 370 original documents using GPT-5.4 and Azure Document Intelligence `prebuilt-layout`, with harness revision `87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91`. - -| Split | Documents | Value F1 | Precision | Recall | ¢ / page | Seconds / document | -|---|---:|---:|---:|---:|---:|---:| -| Short | 252 | 94.29 | 94.04 | 94.90 | 8.42 | 118.8 | -| Medium | 98 | 92.48 | 93.55 | 91.75 | 3.46 | 245.3 | -| Long | 20 | 78.77 | 85.61 | 75.46 | 5.80 | 653.9 | -| Overall | 370 | 92.97 | 93.45 | 93.02 | 4.96 | 181.3 | - -Value scores are document-level macro averages from the pinned official scorer. The harness emits no field citations, so word- and page-grounding scores remain zero. Latency is the mean recorded provider duration per document. - -The full-workload estimate is **$241.53** over 4,869 source pages: **$191.64** for model calls and **$49.89** for Document Intelligence. Prices are $2.50 / $0.25 / $15 per million uncached-input / cached-input / output tokens, with the long-context multipliers applied per request, and $10 per 1,000 Layout pages. Free-tier credits are excluded. - -Pricing starts with a cold prompt cache for each document, then assumes hits for the full unchanged input prefix from earlier completed calls within that document, including conversation history and tool results. It uses a 1,024-token minimum and rounds cached prefixes down to 128-token increments. Cross-document sharing is excluded. Costs are estimates from known usage; unmetered failed requests are additional. - ## Cost of a full run Costs use each provider's official listed price. diff --git a/src/extract_bench/inference/pipelines/extract.py b/src/extract_bench/inference/pipelines/extract.py index 70693064..a0c2de06 100644 --- a/src/extract_bench/inference/pipelines/extract.py +++ b/src/extract_bench/inference/pipelines/extract.py @@ -956,9 +956,7 @@ def register_extract_pipelines(register_fn) -> None: # type: ignore[no-untyped- _pipeline_spec( pipeline_name="kynex_gpt_5_4", provider_name="kynex_extract", - config={"timeout": 3600}, - # The harness enumerates long record lists page by page, so a - # 192-page document needs more than the 600s run default. - per_file_timeout=3600.0, + config={"timeout": 1800}, + per_file_timeout=1800.0, ) ) diff --git a/src/extract_bench/inference/providers/extract/kynex.py b/src/extract_bench/inference/providers/extract/kynex.py index 2bf23e21..44bbe2d7 100644 --- a/src/extract_bench/inference/providers/extract/kynex.py +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -176,11 +176,11 @@ def _run() -> dict[str, Any]: @register_provider("kynex_extract") class KynexExtractProvider(Provider): - """KYNEX extraction with a per-document timeout (default: 3600 seconds).""" + """KYNEX extraction with a per-document timeout (default: 1800 seconds).""" def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None): super().__init__(provider_name, base_config) - self._timeout = int(self.base_config.get("timeout", 3600)) + self._timeout = int(self.base_config.get("timeout", 1800)) def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenceResult: if request.product_type != ProductType.EXTRACT: From 02a63ca98226912f1bc73f101b516ad94dbac12a Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Fri, 30 Oct 2026 00:24:03 -0700 Subject: [PATCH 16/17] Isolate KYNEX workspaces --- src/extract_bench/inference/providers/extract/kynex.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/extract_bench/inference/providers/extract/kynex.py b/src/extract_bench/inference/providers/extract/kynex.py index 44bbe2d7..700f2c0c 100644 --- a/src/extract_bench/inference/providers/extract/kynex.py +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -154,6 +154,8 @@ def _run() -> dict[str, Any]: ] env = os.environ.copy() env.update(HARNESS_ENV) + env["HOME"] = td + env["USERPROFILE"] = td proc = subprocess.run( cmd, capture_output=True, From 8ce16716540b3ef1b5ea341b733f94b5f6c63910 Mon Sep 17 00:00:00 2001 From: Justin Xu Date: Fri, 30 Oct 2026 00:52:27 -0700 Subject: [PATCH 17/17] Scope filelock to KYNEX --- pyproject.toml | 3 ++- uv.lock | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 58cf60eb..3d21d3a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ "autoevals>=0.0.20", "beautifulsoup4>=4.12.0", "bleach>=6.0.0", - "filelock>=3.13", "fire>=0.7.1", "fuzzysearch>=0.7.3", "huggingface-hub>=0.20.0", @@ -67,6 +66,7 @@ runners = [ "datalab-python-sdk", "docling-core>=2.71.0", "extend-ai>=1.16.0", + "filelock>=3.13", "google-cloud-documentai>=2.20.0", "google-genai>=1.0.0", "httpx>=0.28.0", @@ -84,6 +84,7 @@ runners = [ "warp-ingest[ocr]>=2.0.1", ] llamaextract = ["llama-cloud>=1.4.1", "httpx>=0.28.0"] +kynex = ["filelock>=3.13"] openai = ["openai>=1.0.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"] anthropic = ["anthropic>=0.77.1", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"] google = ["google-genai>=1.0.0", "httpx>=0.28.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"] diff --git a/uv.lock b/uv.lock index 7b37147c..8676d7c1 100644 --- a/uv.lock +++ b/uv.lock @@ -1617,7 +1617,6 @@ dependencies = [ { name = "autoevals" }, { name = "beautifulsoup4" }, { name = "bleach" }, - { name = "filelock" }, { name = "fire" }, { name = "fuzzysearch" }, { name = "huggingface-hub" }, @@ -1678,6 +1677,9 @@ google = [ { name = "pymupdf" }, { name = "pypdf" }, ] +kynex = [ + { name = "filelock" }, +] landingai = [ { name = "landingai-ade" }, ] @@ -1718,6 +1720,7 @@ runners = [ { name = "datalab-python-sdk" }, { name = "docling-core" }, { name = "extend-ai" }, + { name = "filelock" }, { name = "google-cloud-documentai" }, { name = "google-genai" }, { name = "httpx" }, @@ -1757,7 +1760,8 @@ requires-dist = [ { name = "docling-core", marker = "extra == 'runners'", specifier = ">=2.71.0" }, { name = "extend-ai", marker = "extra == 'extend'", specifier = ">=1.16.0" }, { name = "extend-ai", marker = "extra == 'runners'", specifier = ">=1.16.0" }, - { name = "filelock", specifier = ">=3.13" }, + { name = "filelock", marker = "extra == 'kynex'", specifier = ">=3.13" }, + { name = "filelock", marker = "extra == 'runners'", specifier = ">=3.13" }, { name = "fire", specifier = ">=0.7.1" }, { name = "fuzzysearch", specifier = ">=0.7.3" }, { name = "google-cloud-documentai", marker = "extra == 'runners'", specifier = ">=2.20.0" }, @@ -1828,7 +1832,7 @@ requires-dist = [ { name = "unstructured-client", marker = "extra == 'runners'", specifier = ">=0.26.0" }, { name = "warp-ingest", extras = ["ocr"], marker = "extra == 'runners'", specifier = ">=2.0.1" }, ] -provides-extras = ["anthropic", "aws", "azure", "datalab", "dev", "extend", "fast", "google", "landingai", "llamaextract", "local", "openai", "openweights", "reducto", "runners"] +provides-extras = ["anthropic", "aws", "azure", "datalab", "dev", "extend", "fast", "google", "kynex", "landingai", "llamaextract", "local", "openai", "openweights", "reducto", "runners"] [[package]] name = "llguidance"