Fail closed on unsafe incremental resumes

README.md

@@ -84,12 +84,18 @@ Targeted allocation is experimental and remains opt-in because it conditions lat
 To avoid wasting judge calls on uninformative pairs, the judge skips comparisons where **both** outputs are shorter than `--min-chars` (default 20 — neither model produced meaningful text), and scores **identical** outputs as an automatic tie without calling the judge. Pass `--min-chars 0` to disable the length filter.
 
 > [!IMPORTANT]
-> If an existing results repository was judged with an older `ocr-bench`
-> checkout from output configs containing multiple appended `inference_info`
-> entries, run the first upgraded evaluation with `--full-rejudge`. Older
-> versions could pair the latest model identity with an earlier OCR text
-> column, and their incremental skip data does not record enough provenance to
-> distinguish those verdicts safely.
+> Results written before per-comparison resume provenance require a one-time
+> `--full-rejudge` or a new results repository. Older versions could pair the
+> latest model identity with an earlier OCR text column when configs contained
+> appended `inference_info` entries; legacy skip data cannot establish which
+> inputs produced a verdict.
+>
+> New comparisons and checkpoints record a fingerprint of the selected source
+> rows/images, OCR contents, resolved model and original output column, judge
+> configuration, and text/image preprocessing settings. Resuming with changed
+> inputs stops before reusing verdicts and requires `--full-rejudge` or a new
+> results repository. Unchanged pairs can still resume when adding a new model,
+> including after an interrupted run that published only a comparisons checkpoint.
 
 **`ocr-bench view`** serves a local web viewer with a leaderboard, comparison browser, and human validation. Vote on comparisons to cross-check the automated judge with human judgement.
 

src/ocr_bench/dataset.py

@@ -2,10 +2,11 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import math
 from dataclasses import dataclass, field
-from typing import Literal
+from typing import Any, Literal
 
 import structlog
 from datasets import Dataset, get_dataset_config_names, load_dataset
@@ -18,6 +19,75 @@ class DatasetError(Exception):
     """Raised when dataset loading or column discovery fails."""
 
 
+def _hash_value(hasher: Any, value: Any) -> None:
+    """Add one typed value to a hashlib-compatible digest deterministically."""
+    update = hasher.update
+    if isinstance(value, bytes):
+        payload = b"bytes:" + value
+    elif hasattr(value, "tobytes") and hasattr(value, "size"):
+        # PIL images: include geometry/mode as well as pixels.
+        payload = (
+            f"image:{getattr(value, 'mode', '')}:{getattr(value, 'size', '')}:".encode()
+            + value.tobytes()
+        )
+    else:
+        payload = json.dumps(
+            value,
+            sort_keys=True,
+            separators=(",", ":"),
+            default=repr,
+        ).encode()
+    update(len(payload).to_bytes(8, "big"))
+    update(payload)
+
+
+def evaluation_input_fingerprints(
+    dataset: Dataset,
+    ocr_columns: dict[str, str],
+    indices: list[int],
+) -> tuple[str, dict[str, str]]:
+    """Fingerprint evaluated pages and each OCR column over the sampled rows.
+
+    The page fingerprint includes stable passthrough identity columns when
+    present and image pixels when available. OCR columns are fingerprinted
+    independently so adding a new model does not invalidate safe comparisons
+    between unchanged existing models.
+    """
+    identity_columns = [key for key in ALIGNMENT_KEYS if key in dataset.column_names]
+    image_columns = [
+        col for col in dataset.column_names if col == "image" or "image" in col.lower()
+    ]
+    if image_columns:
+        # IDs detect reordering; image content also catches a page being replaced
+        # in place while retaining the same ID.
+        image_column = "image" if "image" in image_columns else image_columns[0]
+        identity_columns.append(image_column)
+    if not identity_columns:
+        raise DatasetError(
+            "Cannot fingerprint evaluated pages: no stable identity or image column found"
+        )
+
+    selected = dataset.select(indices)
+    page_digest = hashlib.sha256()
+    for local_idx, source_idx in enumerate(indices):
+        row = selected[local_idx]
+        _hash_value(page_digest, source_idx)
+        for column in identity_columns:
+            _hash_value(page_digest, column)
+            _hash_value(page_digest, row[column])
+
+    column_fingerprints: dict[str, str] = {}
+    for column in ocr_columns:
+        digest = hashlib.sha256()
+        values = selected[column]
+        for source_idx, value in zip(indices, values):
+            _hash_value(digest, source_idx)
+            _hash_value(digest, value)
+        column_fingerprints[column] = digest.hexdigest()
+
+    return page_digest.hexdigest(), column_fingerprints
+
+
 # ---------------------------------------------------------------------------
 # Cross-config row alignment (issue #5)
 # ---------------------------------------------------------------------------
@@ -414,11 +484,14 @@ def load_config_dataset(
     config_names: list[str],
     split: str = "train",
     pr_revisions: dict[str, str] | None = None,
+    *,
+    source_output_columns: dict[str, str] | None = None,
 ) -> tuple[Dataset, dict[str, str]]:
     """Load multiple configs from a Hub dataset and merge into one.
 
     Each config becomes a column whose name is the config name and whose value
-    is the OCR text (from the first column matching heuristics, or ``markdown``).
+    is the OCR text selected together with its model identity from inference
+    metadata, falling back to column heuristics for legacy configs.
 
     Before merging, row alignment is verified across configs on shared
     passthrough columns (see :data:`ALIGNMENT_KEYS`). A mismatch raises
@@ -430,6 +503,8 @@ def load_config_dataset(
         config_names: List of config names to load.
         split: Dataset split to load.
         pr_revisions: Optional mapping of config_name → revision for PR-based loading.
+        source_output_columns: Optional output mapping populated with each loaded
+            config's original OCR column name, before renaming it for the merge.
 
     Returns:
         Tuple of (unified Dataset, {column_name: model_id}).
@@ -497,6 +572,8 @@ def load_config_dataset(
         if text_col is None:  # filtered into `usable` above; narrows the type
             continue
         ocr_columns[config] = lc.model_id
+        if source_output_columns is not None:
+            source_output_columns[config] = text_col
 
         # Build unified dataset using Arrow-level ops (no per-row image decode).
         # Row counts are already verified equal above, so no truncation is needed.

src/ocr_bench/elo.py

@@ -33,6 +33,9 @@ class ComparisonResult:
     col_b: str = ""
     truncated_a: bool = False
     truncated_b: bool = False
+    # Hash of the source/judge/sampling provenance for safe incremental resume.
+    # Historical rows lack it and must not be silently reused.
+    provenance_hash: str = ""
 
 
 @dataclass
@@ -314,6 +317,7 @@ def compute_elo(
                 # flip: truncated_a is always about model_a's output.
                 "truncated_a": r.truncated_a,
                 "truncated_b": r.truncated_b,
+                "provenance_hash": r.provenance_hash,
             }
         )
 

src/ocr_bench/publish.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import datetime
+import hashlib
 import json
 from dataclasses import dataclass, field
 
@@ -19,7 +20,7 @@
     practical_preferences,
 )
 from ocr_bench.elo import ComparisonResult, Leaderboard, compute_elo
-from ocr_bench.judge import MAX_IMAGE_DIM, MAX_OCR_TEXT_LENGTH
+from ocr_bench.judge import DEFAULT_MIN_CHARS, MAX_IMAGE_DIM, MAX_OCR_TEXT_LENGTH
 from ocr_bench.run import MODEL_REGISTRY
 
 logger = structlog.get_logger()
@@ -53,6 +54,16 @@ class EvalMetadata:
     # metadata rows predate this field and are interpreted as ``train`` by the
     # viewer for backward compatibility.
     source_split: str = "train"
+    # Exact loaded-dataset identity and selection used by the resume guard.
+    source_fingerprint: str = ""
+    source_column_fingerprints: dict[str, str] = field(default_factory=dict)
+    source_configs: list[str] = field(default_factory=list)
+    source_columns: dict[str, str] = field(default_factory=dict)
+    # Unified column -> original output column selected from the source config.
+    source_output_columns: dict[str, str] = field(default_factory=dict)
+    # One-way identities of the raw CLI judge specs. Raw endpoint specs may
+    # contain credentials or private hostnames and must never be published.
+    judge_spec_hashes: list[str] = field(default_factory=list)
     auto_tied: int = 0
     # Global comparison budget for the run (--max-comparisons); None = uncapped.
     # ``budget_exhausted`` records whether the run stopped because it hit the cap
@@ -76,6 +87,7 @@ class EvalMetadata:
     # "normalized" (HTML flattened before the cap) or "raw" (capped as-is).
     # Changes verdicts, so it's provenance alongside the caps.
     judge_text_mode: str = "normalized"
+    min_chars: int = DEFAULT_MIN_CHARS
     # Comparison-allocation provenance. ``balanced`` is the historical default;
     # ``targeted`` tops up only unresolved adjacent pairs after enough balanced evidence.
     adaptive_strategy: str = "balanced"
@@ -136,6 +148,7 @@ def load_existing_comparisons(repo_id: str) -> list[ComparisonResult]:
                     col_b=row.get("col_b", ""),
                     truncated_a=row.get("truncated_a", False),
                     truncated_b=row.get("truncated_b", False),
+                    provenance_hash=row.get("provenance_hash", ""),
                 )
             )
     except Exception as exc:
@@ -171,6 +184,56 @@ def _get_model_sizes() -> dict[str, str]:
     return {cfg.model_id: cfg.size for cfg in MODEL_REGISTRY.values()}
 
 
+def evaluation_provenance_hash(
+    *,
+    source_dataset: str,
+    source_split: str,
+    source_fingerprint: str,
+    source_column_fingerprints: dict[str, str],
+    source_columns: dict[str, str],
+    source_output_columns: dict[str, str],
+    judge_specs: list[str],
+    seed: int,
+    max_samples: int,
+    prompt_hash: str,
+    judge_text_mode: str,
+    max_ocr_text_len: int,
+    judge_image_dim: int,
+    max_tokens: int,
+    min_chars: int,
+) -> str:
+    """Hash every input that determines which verdicts are safe to reuse.
+
+    The comparison config carries this hash on every row, including mid-run
+    checkpoints. That lets a killed first run resume safely before a final
+    metadata row exists, while results produced before this guard fail closed.
+    """
+    payload = {
+        "source_dataset": source_dataset,
+        "source_split": source_split,
+        "source_fingerprint": source_fingerprint,
+        "source_column_fingerprints": source_column_fingerprints,
+        "source_columns": source_columns,
+        "source_output_columns": source_output_columns,
+        "judge_specs": judge_specs,
+        "seed": seed,
+        "max_samples": max_samples,
+        "prompt_hash": prompt_hash,
+        "judge_text_mode": judge_text_mode,
+        "max_ocr_text_len": max_ocr_text_len,
+        "judge_image_dim": judge_image_dim,
+        "max_tokens": max_tokens,
+        "min_chars": min_chars,
+    }
+    canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+def hash_judge_specs(judge_specs: list[str]) -> list[str]:
+    """Return publish-safe identities for potentially sensitive judge specs."""
+    return [hashlib.sha256(spec.encode()).hexdigest() for spec in judge_specs]
+
+
 def build_leaderboard_rows(
     board: Leaderboard,
     failed_models: list[str] | None = None,
@@ -238,7 +301,15 @@ def build_metadata_row(metadata: EvalMetadata) -> dict:
     return {
         "source_dataset": metadata.source_dataset,
         "source_split": metadata.source_split,
+        "source_fingerprint": metadata.source_fingerprint,
+        "source_column_fingerprints": json.dumps(
+            metadata.source_column_fingerprints, sort_keys=True
+        ),
+        "source_configs": json.dumps(metadata.source_configs),
+        "source_columns": json.dumps(metadata.source_columns, sort_keys=True),
+        "source_output_columns": json.dumps(metadata.source_output_columns, sort_keys=True),
         "judge_models": json.dumps(metadata.judge_models),
+        "judge_spec_hashes": json.dumps(metadata.judge_spec_hashes),
         "seed": metadata.seed,
         "max_samples": metadata.max_samples,
         "total_comparisons": metadata.total_comparisons,
@@ -255,6 +326,7 @@ def build_metadata_row(metadata: EvalMetadata) -> dict:
         "max_ocr_text_len": metadata.max_ocr_text_len,
         "judge_image_dim": metadata.judge_image_dim,
         "judge_text_mode": metadata.judge_text_mode,
+        "min_chars": metadata.min_chars,
         "adaptive_strategy": metadata.adaptive_strategy,
         "size_tie_ratio": metadata.size_tie_ratio,
         "size_tie_min_samples": metadata.size_tie_min_samples,