diff --git a/README.md b/README.md index 07d82f2d..8d8b10f8 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ In the official benchmark, documents that fail or exceed a 30-minute limit score | 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ | | 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ | -Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv). +Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). @@ -89,7 +89,7 @@ Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv). -Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv). +Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv). ### ExtractBench Pro leaderboard diff --git a/leaderboard.csv b/leaderboard.csv index 8f34f1ea..aaa425d5 100644 --- a/leaderboard.csv +++ b/leaderboard.csv @@ -9,6 +9,7 @@ Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813 Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0 Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6 Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3 +KYNEX (gpt-5.4),Coding Agents,92.97,94.29,92.48,78.77,0.0496,0.0842,0.0346,0.0580,94.04,93.55,85.61,94.90,91.75,75.46,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,118.8,245.3,653.9 Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3 OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1 Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1 diff --git a/pyproject.toml b/pyproject.toml index 68c717a3..3d21d3a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ runners = [ "datalab-python-sdk", "docling-core>=2.71.0", "extend-ai>=1.16.0", + "filelock>=3.13", "google-cloud-documentai>=2.20.0", "google-genai>=1.0.0", "httpx>=0.28.0", @@ -83,6 +84,7 @@ runners = [ "warp-ingest[ocr]>=2.0.1", ] llamaextract = ["llama-cloud>=1.4.1", "httpx>=0.28.0"] +kynex = ["filelock>=3.13"] openai = ["openai>=1.0.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"] anthropic = ["anthropic>=0.77.1", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"] google = ["google-genai>=1.0.0", "httpx>=0.28.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"] diff --git a/src/extract_bench/inference/pipelines/extract.py b/src/extract_bench/inference/pipelines/extract.py index 77b3073c..a0c2de06 100644 --- a/src/extract_bench/inference/pipelines/extract.py +++ b/src/extract_bench/inference/pipelines/extract.py @@ -950,3 +950,13 @@ def register_extract_pipelines(register_fn) -> None: # type: ignore[no-untyped- }, ) ) + + # The provider provisions a pinned KYNEX checkout and isolated dependencies. + register_fn( + _pipeline_spec( + pipeline_name="kynex_gpt_5_4", + provider_name="kynex_extract", + config={"timeout": 1800}, + per_file_timeout=1800.0, + ) + ) diff --git a/src/extract_bench/inference/providers/extract/__init__.py b/src/extract_bench/inference/providers/extract/__init__.py index 968118e7..1aa2c6ad 100644 --- a/src/extract_bench/inference/providers/extract/__init__.py +++ b/src/extract_bench/inference/providers/extract/__init__.py @@ -32,6 +32,7 @@ "llamaextract_v2_api", "nuextract3_extract", "openai_responses", + "kynex", "pulse", "reducto", "table_codegen_extract", diff --git a/src/extract_bench/inference/providers/extract/kynex.py b/src/extract_bench/inference/providers/extract/kynex.py new file mode 100644 index 00000000..700f2c0c --- /dev/null +++ b/src/extract_bench/inference/providers/extract/kynex.py @@ -0,0 +1,245 @@ +"""Run a pinned KYNEX harness in an automatically provisioned virtual environment. + +The first extraction clones https://github.com/Nikit117/kynex-harness and +installs its requirements into an isolated cache. Later extractions reuse that +installation. Set KYNEX_CACHE_DIR to choose the cache location; Azure OpenAI +and Document Intelligence credentials come from the process environment. + +The upstream agent and post-processing rules run unchanged. No per-field +citation boxes are emitted, so field_citations is empty. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from datetime import datetime +from pathlib import Path +from typing import Any + +from filelock import FileLock, Timeout + +from extract_bench.inference.providers.base import ( + Provider, + ProviderConfigError, + ProviderPermanentError, +) +from extract_bench.inference.providers.registry import register_provider +from extract_bench.schemas.extract_output import ExtractOutput +from extract_bench.schemas.pipeline_io import ( + InferenceRequest, + InferenceResult, + RawInferenceResult, +) +from extract_bench.schemas.product import ProductType + +logger = logging.getLogger(__name__) + +HARNESS_REPOSITORY = "https://github.com/Nikit117/kynex-harness.git" +HARNESS_REVISION = "87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91" +_SETUP_TIMEOUT_S = 900 + +# Preserve the rule switches from the submitted benchmark configuration. +HARNESS_ENV: dict[str, str] = { + "ADH_BULK_ARRAYS": "1", + "ADH_PRUNE_NON_RECORDS": "1", + "ADH_REDACTION_MARKERS": "1", + "ADH_OCR_FALLBACK": "1", + "ADH_BULK_OCR": "1", + "ADH_STRUCTURED_ARRAYS": "1", +} + + +def _run_setup(command: Sequence[str], *, cwd: Path | None = None) -> str: + try: + result = subprocess.run( + command, + cwd=cwd, + check=True, + capture_output=True, + text=True, + timeout=_SETUP_TIMEOUT_S, + ) + except subprocess.CalledProcessError as exc: + detail = exc.stderr if exc.stderr is not None and len(exc.stderr) > 0 else exc.stdout + if detail is None: + detail = str(exc) + raise ProviderConfigError(f"KYNEX setup failed: {detail[-2000:]}") from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise ProviderConfigError(f"KYNEX setup failed: {exc}") from exc + return result.stdout.strip() + + +def _ensure_harness() -> tuple[Path, Path]: + """Publish a complete pinned installation under a process-shared lock.""" + root = Path(os.environ.get("KYNEX_CACHE_DIR", str(Path.home() / ".cache" / "extract-bench" / "kynex"))) + cache = root.expanduser().resolve() / HARNESS_REVISION / f"py{sys.version_info.major}.{sys.version_info.minor}" + cache.mkdir(parents=True, exist_ok=True) + repo = cache / "repo" + environment = cache / "venv" + python = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + ready = cache / "ready" + try: + with FileLock(cache / "setup.lock", timeout=_SETUP_TIMEOUT_S): + if not repo.exists(): + logger.info("Cloning KYNEX at %s", HARNESS_REVISION) + with tempfile.TemporaryDirectory(prefix="clone-", dir=cache) as staging: + checkout = Path(staging) / "repo" + _run_setup( + [ + "git", + "clone", + "--no-checkout", + HARNESS_REPOSITORY, + str(checkout), + ] + ) + _run_setup(["git", "checkout", "--detach", HARNESS_REVISION], cwd=checkout) + checkout.rename(repo) + actual_revision = _run_setup(["git", "rev-parse", "HEAD"], cwd=repo) + changes = _run_setup(["git", "status", "--porcelain"], cwd=repo) + if actual_revision != HARNESS_REVISION or len(changes) > 0: + raise ProviderConfigError(f"Cached KYNEX checkout differs from {HARNESS_REVISION}: {repo}") + if ready.exists() and python.exists() and ready.read_text(encoding="utf-8") == HARNESS_REVISION: + return repo, python + ready.unlink(missing_ok=True) + logger.info("Installing KYNEX dependencies in %s", environment) + _run_setup([sys.executable, "-m", "venv", str(environment)]) + _run_setup( + [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + "--requirement", + str(repo / "requirements.txt"), + ], + cwd=repo, + ) + ready.write_text(HARNESS_REVISION, encoding="utf-8") + except Timeout as exc: + raise ProviderConfigError(f"Timed out waiting for KYNEX setup: {cache}") from exc + return repo, python + + +async def _extract_async( + source: Path, schema: dict[str, Any], timeout: float, repo: Path, python: Path +) -> dict[str, Any]: + """Run one extraction in the managed harness environment.""" + + def _run() -> dict[str, Any]: + script = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" + with tempfile.TemporaryDirectory() as td: + schema_file = Path(td) / "schema.json" + schema_file.write_text(json.dumps(schema), encoding="utf-8") + out_file = Path(td) / "extraction.json" + cmd = [ + str(python), + "-I", + str(script), + "--source", + str(source), + "--schema", + str(schema_file), + "--out", + str(out_file), + ] + env = os.environ.copy() + env.update(HARNESS_ENV) + env["HOME"] = td + env["USERPROFILE"] = td + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=str(repo), + timeout=timeout, + env=env, + ) + if proc.returncode != 0: + raise ProviderPermanentError( + f"harness extraction failed (exit {proc.returncode}): {(proc.stderr or proc.stdout)[-400:]}" + ) + if not out_file.exists(): + raise ProviderPermanentError("harness produced no extraction file") + extraction = json.loads(out_file.read_text(encoding="utf-8")) + return {"extraction": extraction, "harness_revision": HARNESS_REVISION} + + return await asyncio.get_running_loop().run_in_executor(None, _run) + + +@register_provider("kynex_extract") +class KynexExtractProvider(Provider): + """KYNEX extraction with a per-document timeout (default: 1800 seconds).""" + + def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None): + super().__init__(provider_name, base_config) + self._timeout = int(self.base_config.get("timeout", 1800)) + + def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenceResult: + if request.product_type != ProductType.EXTRACT: + raise ProviderPermanentError( + f"KynexExtractProvider only supports EXTRACT product type, got {request.product_type}" + ) + if not request.schema_override: + raise ProviderPermanentError("schema_override is required for EXTRACT product type.") + source = Path(request.source_file_path).resolve() + if not source.exists(): + raise ProviderPermanentError(f"File not found: {source}") + + repo, python = _ensure_harness() + started_at = datetime.now() + try: + raw_output = asyncio.run( + asyncio.wait_for( + _extract_async(source, request.schema_override, self._timeout, repo, python), + timeout=self._timeout, + ) + ) + except TimeoutError as exc: + raise ProviderPermanentError(f"harness extraction timed out after {self._timeout}s") from exc + except ProviderPermanentError: + raise + except Exception as exc: # noqa: BLE001 -- report as a document failure + raise ProviderPermanentError(f"harness extraction failed: {exc}") from exc + + completed_at = datetime.now() + return RawInferenceResult( + request=request, + pipeline=pipeline, + pipeline_name=pipeline.pipeline_name, + product_type=request.product_type, + raw_output=raw_output, + started_at=started_at, + completed_at=completed_at, + latency_in_ms=int((completed_at - started_at).total_seconds() * 1000), + ) + + def normalize(self, raw_result: RawInferenceResult) -> InferenceResult: + extraction = raw_result.raw_output.get("extraction") + extracted_data = extraction if isinstance(extraction, (dict, list)) else {} + output = ExtractOutput( + task_type="extract", + example_id=raw_result.request.example_id, + pipeline_name=raw_result.pipeline_name, + extracted_data=extracted_data, + field_citations=[], + ) + return InferenceResult( + request=raw_result.request, + pipeline_name=raw_result.pipeline_name, + product_type=raw_result.product_type, + raw_output=raw_result.raw_output, + output=output, + started_at=raw_result.started_at, + completed_at=raw_result.completed_at, + latency_in_ms=raw_result.latency_in_ms, + ) diff --git a/tests/extract_bench/inference/providers/extract/test_kynex.py b/tests/extract_bench/inference/providers/extract/test_kynex.py new file mode 100644 index 00000000..fda3fddc --- /dev/null +++ b/tests/extract_bench/inference/providers/extract/test_kynex.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import subprocess +import sys +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from zipfile import ZipFile + +import pytest + +from extract_bench.inference.pipelines import get_pipeline +from extract_bench.inference.providers.base import ProviderConfigError +from extract_bench.inference.providers.extract import kynex +from extract_bench.inference.providers.registry import create_provider +from extract_bench.schemas.pipeline_io import InferenceRequest +from extract_bench.schemas.product import ProductType + + +def _git(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + +@pytest.fixture +def harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, str]: + """A local harness with a real installable dependency and a newer, wrong HEAD.""" + repo = tmp_path / "upstream" + repo.mkdir() + _git(repo, "init") + _git(repo, "config", "user.email", "fixture@example.invalid") + _git(repo, "config", "user.name", "Test fixture") + wheel = repo / "fixture_dependency-1.0-py3-none-any.whl" + with ZipFile(wheel, "w") as archive: + files = { + "fixture_dependency/__init__.py": 'VALUE = "installed dependency"\n', + "fixture_dependency-1.0.dist-info/METADATA": ( + "Metadata-Version: 2.1\nName: fixture-dependency\nVersion: 1.0\n" + ), + "fixture_dependency-1.0.dist-info/WHEEL": ( + "Wheel-Version: 1.0\nGenerator: fixture\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + ), + } + for name, text in files.items(): + archive.writestr(name, text) + record = "fixture_dependency-1.0.dist-info/RECORD" + archive.writestr(record, "".join(f"{name},,\n" for name in [*files, record])) + (repo / "requirements.txt").write_text(f"./{wheel.name}\n", encoding="utf-8") + entry = repo / "autonomous_document_harness" / "scripts" / "extract_one.py" + entry.parent.mkdir(parents=True) + entry.write_text( + "import argparse, json\n" + "from pathlib import Path\n" + "from fixture_dependency import VALUE\n" + "p = argparse.ArgumentParser()\n" + "for name in ('source', 'schema', 'out'): p.add_argument('--' + name, required=True)\n" + "a = p.parse_args()\n" + "assert json.loads(Path(a.schema).read_text())['type'] == 'object'\n" + "Path(a.out).write_text(json.dumps({'value': VALUE, 'source': Path(a.source).read_text()}))\n", + encoding="utf-8", + ) + _git(repo, "add", ".") + _git(repo, "commit", "-m", "Pinned harness") + revision = _git(repo, "rev-parse", "HEAD") + entry.write_text("raise RuntimeError('wrong revision')\n", encoding="utf-8") + _git(repo, "commit", "-am", "Newer upstream revision must not run") + monkeypatch.setattr(kynex, "HARNESS_REPOSITORY", str(repo)) + monkeypatch.setattr(kynex, "HARNESS_REVISION", revision) + monkeypatch.setenv("KYNEX_CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setenv("PIP_NO_INDEX", "1") + return repo, revision + + +def test_registered_pipeline_installs_pinned_harness_once_for_concurrent_documents( + harness: tuple[Path, str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Catch missing registration, running upstream HEAD, missing deps, and setup races.""" + pipeline = get_pipeline("kynex_gpt_5_4") + provider = create_provider(pipeline) + setup_commands: list[Sequence[str]] = [] + original = kynex._run_setup + + def track(command: Sequence[str], *, cwd: Path | None = None) -> str: + setup_commands.append(command) + return original(command, cwd=cwd) + + monkeypatch.setattr(kynex, "_run_setup", track) + with ThreadPoolExecutor(max_workers=2) as pool: + installations = list(pool.map(lambda _: kynex._ensure_harness(), range(2))) + repo, python = installations[0] + assert installations[1] == (repo, python) + assert _git(repo, "rev-parse", "HEAD") == harness[1] + assert python != Path(sys.executable) + monkeypatch.chdir(tmp_path) + (tmp_path / "invoice.pdf").write_text("document input", encoding="utf-8") + request = InferenceRequest( + example_id="fixture", + source_file_path="invoice.pdf", + product_type=ProductType.EXTRACT, + schema_override={"type": "object", "properties": {"value": {"type": "string"}}}, + ) + raw = provider.run_inference(pipeline, request) + output = provider.normalize(raw) + assert output.output.extracted_data == { + "value": "installed dependency", + "source": "document input", + } + assert raw.raw_output["harness_revision"] == harness[1] + assert sum("clone" in command for command in setup_commands) == 1 + assert sum("pip" in command for command in setup_commands) == 1 + + (repo / "requirements.txt").write_text("unexpected edit\n", encoding="utf-8") + with pytest.raises(ProviderConfigError, match="differs from"): + kynex._ensure_harness() + + +def test_failed_dependency_install_is_retried_instead_of_reused_as_ready( + harness: tuple[Path, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """An interrupted/failed installation must not publish a reusable environment.""" + original = kynex._run_setup + + def fail_install(command: Sequence[str], *, cwd: Path | None = None) -> str: + if "pip" in command: + raise ProviderConfigError("dependency install interrupted") + return original(command, cwd=cwd) + + monkeypatch.setattr(kynex, "_run_setup", fail_install) + with pytest.raises(ProviderConfigError, match="dependency install interrupted"): + kynex._ensure_harness() + monkeypatch.setattr(kynex, "_run_setup", original) + repo, python = kynex._ensure_harness() + assert _git(repo, "rev-parse", "HEAD") == harness[1] + result = subprocess.check_output( + [ + str(python), + "-c", + "import fixture_dependency; print(fixture_dependency.VALUE)", + ], + text=True, + ) + assert result.strip() == "installed dependency" diff --git a/uv.lock b/uv.lock index d04997e2..8676d7c1 100644 --- a/uv.lock +++ b/uv.lock @@ -1677,6 +1677,9 @@ google = [ { name = "pymupdf" }, { name = "pypdf" }, ] +kynex = [ + { name = "filelock" }, +] landingai = [ { name = "landingai-ade" }, ] @@ -1717,6 +1720,7 @@ runners = [ { name = "datalab-python-sdk" }, { name = "docling-core" }, { name = "extend-ai" }, + { name = "filelock" }, { name = "google-cloud-documentai" }, { name = "google-genai" }, { name = "httpx" }, @@ -1756,6 +1760,8 @@ requires-dist = [ { name = "docling-core", marker = "extra == 'runners'", specifier = ">=2.71.0" }, { name = "extend-ai", marker = "extra == 'extend'", specifier = ">=1.16.0" }, { name = "extend-ai", marker = "extra == 'runners'", specifier = ">=1.16.0" }, + { name = "filelock", marker = "extra == 'kynex'", specifier = ">=3.13" }, + { name = "filelock", marker = "extra == 'runners'", specifier = ">=3.13" }, { name = "fire", specifier = ">=0.7.1" }, { name = "fuzzysearch", specifier = ">=0.7.3" }, { name = "google-cloud-documentai", marker = "extra == 'runners'", specifier = ">=2.20.0" }, @@ -1826,7 +1832,7 @@ requires-dist = [ { name = "unstructured-client", marker = "extra == 'runners'", specifier = ">=0.26.0" }, { name = "warp-ingest", extras = ["ocr"], marker = "extra == 'runners'", specifier = ">=2.0.1" }, ] -provides-extras = ["anthropic", "aws", "azure", "datalab", "dev", "extend", "fast", "google", "landingai", "llamaextract", "local", "openai", "openweights", "reducto", "runners"] +provides-extras = ["anthropic", "aws", "azure", "datalab", "dev", "extend", "fast", "google", "kynex", "landingai", "llamaextract", "local", "openai", "openweights", "reducto", "runners"] [[package]] name = "llguidance"