Add KYNEX (gpt-5.4) to the ExtractBench leaderboard

README.md

@@ -64,7 +64,7 @@ In the official benchmark, documents that fail or exceed a 30-minute limit score
 | 9 | Codex (GPT-6.1 Terra Evidence) | Coding Agents | 94.28 | 96.26 | 91.86 | 81.21 | 4.68¢ |
 | 10 | LlamaExtract Cost-Effective | LlamaExtract | 94.21 | 95.49 | 91.58 | 90.86 | 1.00¢ |
 
-Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv).
+Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv).
 <!-- LEADERBOARD:END -->
 
 <!-- GROUNDING:START -->
@@ -89,7 +89,7 @@ Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv).
   </tbody>
 </table>
 
-Top 10 of 51 systems — full table in [leaderboard.csv](leaderboard.csv).
+Top 10 of 52 systems — full table in [leaderboard.csv](leaderboard.csv).
 <!-- GROUNDING:END -->
 
 ### ExtractBench Pro leaderboard

leaderboard.csv

@@ -9,6 +9,7 @@ Codex (GPT-5.6 Sol Evidence),Coding Agents,93.77,96.04,90.19,82.73,0.2165,0.2813
 Codex (GPT-5.5),Coding Agents,93.57,95.68,91.15,78.88,0.2783,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0
 Codex (GPT-5.5 Evidence),Coding Agents,93.35,95.59,88.72,87.97,0.3363,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6
 Pulse (Non-effort),Specialized APIs,94.51,95.39,92.49,93.31,0.0300,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3
+KYNEX (gpt-5.4),Coding Agents,92.97,94.29,92.48,78.77,0.0496,0.0842,0.0346,0.0580,94.04,93.55,85.61,94.90,91.75,75.46,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,118.8,245.3,653.9
 Codex (GPT-5.6 Terra Evidence),Coding Agents,92.26,95.60,86.69,77.36,0.0945,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3
 OpenAI GPT-6 Astra,Commercial VLM,91.91,97.22,90.56,31.70,0.1109,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1
 Codex (GPT-5.6 Luna Evidence),Coding Agents,91.16,93.50,87.43,80.05,0.0120,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1

pyproject.toml

@@ -66,6 +66,7 @@ runners = [
     "datalab-python-sdk",
     "docling-core>=2.71.0",
     "extend-ai>=1.16.0",
+    "filelock>=3.13",
     "google-cloud-documentai>=2.20.0",
     "google-genai>=1.0.0",
     "httpx>=0.28.0",
@@ -83,6 +84,7 @@ runners = [
     "warp-ingest[ocr]>=2.0.1",
 ]
 llamaextract = ["llama-cloud>=1.4.1", "httpx>=0.28.0"]
+kynex = ["filelock>=3.13"]
 openai = ["openai>=1.0.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]
 anthropic = ["anthropic>=0.77.1", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]
 google = ["google-genai>=1.0.0", "httpx>=0.28.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]

src/extract_bench/inference/pipelines/extract.py

@@ -950,3 +950,13 @@ def register_extract_pipelines(register_fn) -> None:  # type: ignore[no-untyped-
             },
         )
     )
+
+    # The provider provisions a pinned KYNEX checkout and isolated dependencies.
+    register_fn(
+        _pipeline_spec(
+            pipeline_name="kynex_gpt_5_4",
+            provider_name="kynex_extract",
+            config={"timeout": 1800},
+            per_file_timeout=1800.0,
+        )
+    )

src/extract_bench/inference/providers/extract/__init__.py

@@ -32,6 +32,7 @@
     "llamaextract_v2_api",
     "nuextract3_extract",
     "openai_responses",
+    "kynex",
     "pulse",
     "reducto",
     "table_codegen_extract",

src/extract_bench/inference/providers/extract/kynex.py

@@ -0,0 +1,245 @@
+"""Run a pinned KYNEX harness in an automatically provisioned virtual environment.
+
+The first extraction clones https://github.com/Nikit117/kynex-harness and
+installs its requirements into an isolated cache. Later extractions reuse that
+installation. Set KYNEX_CACHE_DIR to choose the cache location; Azure OpenAI
+and Document Intelligence credentials come from the process environment.
+
+The upstream agent and post-processing rules run unchanged. No per-field
+citation boxes are emitted, so field_citations is empty.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+from collections.abc import Sequence
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from filelock import FileLock, Timeout
+
+from extract_bench.inference.providers.base import (
+    Provider,
+    ProviderConfigError,
+    ProviderPermanentError,
+)
+from extract_bench.inference.providers.registry import register_provider
+from extract_bench.schemas.extract_output import ExtractOutput
+from extract_bench.schemas.pipeline_io import (
+    InferenceRequest,
+    InferenceResult,
+    RawInferenceResult,
+)
+from extract_bench.schemas.product import ProductType
+
+logger = logging.getLogger(__name__)
+
+HARNESS_REPOSITORY = "https://github.com/Nikit117/kynex-harness.git"
+HARNESS_REVISION = "87a7315f4fb9c4e499f1c0599ff6d8d4d2b0ae91"
+_SETUP_TIMEOUT_S = 900
+
+# Preserve the rule switches from the submitted benchmark configuration.
+HARNESS_ENV: dict[str, str] = {
+    "ADH_BULK_ARRAYS": "1",
+    "ADH_PRUNE_NON_RECORDS": "1",
+    "ADH_REDACTION_MARKERS": "1",
+    "ADH_OCR_FALLBACK": "1",
+    "ADH_BULK_OCR": "1",
+    "ADH_STRUCTURED_ARRAYS": "1",
+}
+
+
+def _run_setup(command: Sequence[str], *, cwd: Path | None = None) -> str:
+    try:
+        result = subprocess.run(
+            command,
+            cwd=cwd,
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=_SETUP_TIMEOUT_S,
+        )
+    except subprocess.CalledProcessError as exc:
+        detail = exc.stderr if exc.stderr is not None and len(exc.stderr) > 0 else exc.stdout
+        if detail is None:
+            detail = str(exc)
+        raise ProviderConfigError(f"KYNEX setup failed: {detail[-2000:]}") from exc
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        raise ProviderConfigError(f"KYNEX setup failed: {exc}") from exc
+    return result.stdout.strip()
+
+
+def _ensure_harness() -> tuple[Path, Path]:
+    """Publish a complete pinned installation under a process-shared lock."""
+    root = Path(os.environ.get("KYNEX_CACHE_DIR", str(Path.home() / ".cache" / "extract-bench" / "kynex")))
+    cache = root.expanduser().resolve() / HARNESS_REVISION / f"py{sys.version_info.major}.{sys.version_info.minor}"
+    cache.mkdir(parents=True, exist_ok=True)
+    repo = cache / "repo"
+    environment = cache / "venv"
+    python = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+    ready = cache / "ready"
+    try:
+        with FileLock(cache / "setup.lock", timeout=_SETUP_TIMEOUT_S):
+            if not repo.exists():
+                logger.info("Cloning KYNEX at %s", HARNESS_REVISION)
+                with tempfile.TemporaryDirectory(prefix="clone-", dir=cache) as staging:
+                    checkout = Path(staging) / "repo"
+                    _run_setup(
+                        [
+                            "git",
+                            "clone",
+                            "--no-checkout",
+                            HARNESS_REPOSITORY,
+                            str(checkout),
+                        ]
+                    )
+                    _run_setup(["git", "checkout", "--detach", HARNESS_REVISION], cwd=checkout)
+                    checkout.rename(repo)
+            actual_revision = _run_setup(["git", "rev-parse", "HEAD"], cwd=repo)
+            changes = _run_setup(["git", "status", "--porcelain"], cwd=repo)
+            if actual_revision != HARNESS_REVISION or len(changes) > 0:
+                raise ProviderConfigError(f"Cached KYNEX checkout differs from {HARNESS_REVISION}: {repo}")
+            if ready.exists() and python.exists() and ready.read_text(encoding="utf-8") == HARNESS_REVISION:
+                return repo, python
+            ready.unlink(missing_ok=True)
+            logger.info("Installing KYNEX dependencies in %s", environment)
+            _run_setup([sys.executable, "-m", "venv", str(environment)])
+            _run_setup(
+                [
+                    str(python),
+                    "-m",
+                    "pip",
+                    "install",
+                    "--disable-pip-version-check",
+                    "--no-input",
+                    "--requirement",
+                    str(repo / "requirements.txt"),
+                ],
+                cwd=repo,
+            )
+            ready.write_text(HARNESS_REVISION, encoding="utf-8")
+    except Timeout as exc:
+        raise ProviderConfigError(f"Timed out waiting for KYNEX setup: {cache}") from exc
+    return repo, python
+
+
+async def _extract_async(
+    source: Path, schema: dict[str, Any], timeout: float, repo: Path, python: Path
+) -> dict[str, Any]:
+    """Run one extraction in the managed harness environment."""
+
+    def _run() -> dict[str, Any]:
+        script = repo / "autonomous_document_harness" / "scripts" / "extract_one.py"
+        with tempfile.TemporaryDirectory() as td:
+            schema_file = Path(td) / "schema.json"
+            schema_file.write_text(json.dumps(schema), encoding="utf-8")
+            out_file = Path(td) / "extraction.json"
+            cmd = [
+                str(python),
+                "-I",
+                str(script),
+                "--source",
+                str(source),
+                "--schema",
+                str(schema_file),
+                "--out",
+                str(out_file),
+            ]
+            env = os.environ.copy()
+            env.update(HARNESS_ENV)
+            env["HOME"] = td
+            env["USERPROFILE"] = td
+            proc = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                cwd=str(repo),
+                timeout=timeout,
+                env=env,
+            )
+            if proc.returncode != 0:
+                raise ProviderPermanentError(
+                    f"harness extraction failed (exit {proc.returncode}): {(proc.stderr or proc.stdout)[-400:]}"
+                )
+            if not out_file.exists():
+                raise ProviderPermanentError("harness produced no extraction file")
+            extraction = json.loads(out_file.read_text(encoding="utf-8"))
+        return {"extraction": extraction, "harness_revision": HARNESS_REVISION}
+
+    return await asyncio.get_running_loop().run_in_executor(None, _run)
+
+
+@register_provider("kynex_extract")
+class KynexExtractProvider(Provider):
+    """KYNEX extraction with a per-document timeout (default: 1800 seconds)."""
+
+    def __init__(self, provider_name: str, base_config: dict[str, Any] | None = None):
+        super().__init__(provider_name, base_config)
+        self._timeout = int(self.base_config.get("timeout", 1800))
+
+    def run_inference(self, pipeline: Any, request: InferenceRequest) -> RawInferenceResult:
+        if request.product_type != ProductType.EXTRACT:
+            raise ProviderPermanentError(
+                f"KynexExtractProvider only supports EXTRACT product type, got {request.product_type}"
+            )
+        if not request.schema_override:
+            raise ProviderPermanentError("schema_override is required for EXTRACT product type.")
+        source = Path(request.source_file_path).resolve()
+        if not source.exists():
+            raise ProviderPermanentError(f"File not found: {source}")
+
+        repo, python = _ensure_harness()
+        started_at = datetime.now()
+        try:
+            raw_output = asyncio.run(
+                asyncio.wait_for(
+                    _extract_async(source, request.schema_override, self._timeout, repo, python),
+                    timeout=self._timeout,
+                )
+            )
+        except TimeoutError as exc:
+            raise ProviderPermanentError(f"harness extraction timed out after {self._timeout}s") from exc
+        except ProviderPermanentError:
+            raise
+        except Exception as exc:  # noqa: BLE001 -- report as a document failure
+            raise ProviderPermanentError(f"harness extraction failed: {exc}") from exc
+
+        completed_at = datetime.now()
+        return RawInferenceResult(
+            request=request,
+            pipeline=pipeline,
+            pipeline_name=pipeline.pipeline_name,
+            product_type=request.product_type,
+            raw_output=raw_output,
+            started_at=started_at,
+            completed_at=completed_at,
+            latency_in_ms=int((completed_at - started_at).total_seconds() * 1000),
+        )
+
+    def normalize(self, raw_result: RawInferenceResult) -> InferenceResult:
+        extraction = raw_result.raw_output.get("extraction")
+        extracted_data = extraction if isinstance(extraction, (dict, list)) else {}
+        output = ExtractOutput(
+            task_type="extract",
+            example_id=raw_result.request.example_id,
+            pipeline_name=raw_result.pipeline_name,
+            extracted_data=extracted_data,
+            field_citations=[],
+        )
+        return InferenceResult(
+            request=raw_result.request,
+            pipeline_name=raw_result.pipeline_name,
+            product_type=raw_result.product_type,
+            raw_output=raw_result.raw_output,
+            output=output,
+            started_at=raw_result.started_at,
+            completed_at=raw_result.completed_at,
+            latency_in_ms=raw_result.latency_in_ms,
+        )