From 4a9570f7243a4a60cbcd786473324aed64b53d4e Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:11:17 +0800 Subject: [PATCH 1/3] fix(webfetch): qualify live bodies without replacing frozen references --- .agents/skills/benchmark-report/SKILL.md | 2 +- AGENTS.md | 2 +- harness/web_fetch/README.md | 13 +- harness/web_fetch/reference_freeze.py | 17 +- harness/web_fetch/reference_lifecycle.py | 48 ++++- tests/test_live_body_qualification.py | 262 +++++++++++++++++++++++ 6 files changed, 335 insertions(+), 9 deletions(-) create mode 100644 tests/test_live_body_qualification.py diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md index ecdf1b130..05388a346 100644 --- a/.agents/skills/benchmark-report/SKILL.md +++ b/.agents/skills/benchmark-report/SKILL.md @@ -37,7 +37,7 @@ Version changes require an explicit human release request. Code synchronization, Public report CLIs fall back to English. `BROWSER_EVAL_REPORT_LANGUAGE` changes the environment default; an explicit `--language` wins, and a rebuild preserves the saved primary language. For an agent-driven build, select the language from the current conversation. English builds retain the matching Chinese edition; a Chinese selection does not generate an English translation unless explicitly requested. Keep measured material in its source language. Historical Chinese-only renderers must disclose that limit and reject unsupported requests. Bind generated languages and the selected entry in report metadata. English navigation opens a not-generated notice when absent; never publish stale English pages. Validate each generated edition and retain route, value and evidence-link parity checks when both exist. 1. Establish the decision being supported, the measured unit, cohort, versions, execution location, freshness policy, and intended deliverable. Inspect the actual dataset, runner, outputs, scorer, prior review evidence, and current report. Do not infer them from screenshots or prose. -2. Freeze a machine-readable evaluation contract before expensive calls. It must identify the exact cases and methods, run count, timeout, concurrency policy, input and output contract, scoring provenance, reference state, and publication target when one exists. Existing compiled reference answers are frozen by default. Source metadata, capture dates, and absent newer acceptance-contract fields do not imply a live-reference task. Only tasks explicitly declaring `source_verification.reference_mode: live_exact` require live checks. For those tasks, fail closed: verify and freeze references immediately before calls, verify the same declared scope afterward, and certify the cohort only when both checks pass. An unchanged compiled acceptance-contract dataset is already the benchmark input: freeze its task and rule bytes for execution, without requiring historical source-qualification raw files or re-auditing cohort admission. Qualify sources when adding tasks or changing references/acceptance rules; do not turn an ordinary version rerun into dataset reconstruction. Partial or uncertified cohorts remain diagnostic evidence and cannot enter a release. +2. Freeze a machine-readable evaluation contract before expensive calls. It must identify the exact cases and methods, run count, timeout, concurrency policy, input and output contract, scoring provenance, reference state, and publication target when one exists. Existing compiled reference answers are frozen by default. Source metadata, capture dates, and absent newer acceptance-contract fields do not imply a live-reference task. Only tasks explicitly declaring `source_verification.reference_mode: live_exact` or `live_body` require live checks. `live_body` applies the frozen body contract to independent pre/post source text without replacing the reference. For those tasks, fail closed: verify and freeze references immediately before calls, verify the same declared scope afterward, and certify the cohort only when both checks pass. An unchanged compiled acceptance-contract dataset is already the benchmark input: freeze its task and rule bytes for execution, without requiring historical source-qualification raw files or re-auditing cohort admission. Qualify sources when adding tasks or changing references/acceptance rules; do not turn an ordinary version rerun into dataset reconstruction. Partial or uncertified cohorts remain diagnostic evidence and cannot enter a release. 3. Let each method use its strongest documented route to produce the required output. Equalize the task, inputs, output contract, call budget, and scoring. Do not require identical internals. For fixed-native cohorts, declare the input media type independently of URL patterns and expected answers, freeze the class-level route and transport parameters before calls, and verify the retained request receipts at aggregation and release. Record material asymmetries such as local versus remote execution, cold versus warm start, bundled conversion, network transit, account tier, and billable time. 4. Preserve evidence as an append-only chain by default: task definition, reference, provider call, raw artifact, normalized artifact, score, review, revision, and release. If the user explicitly requests deletion of superseded history, validate the chain first and use the existing owner's current-only authority described in [audit and revision](references/audit-and-revision.md); remove obsolete full snapshots rather than storing reverse patches. Keep current provider outputs, call identities, real scoring sources and necessary review evidence immutable. Routine refresh and publication reuse current inputs without redundant historical snapshots. A presentation or scoring change must not masquerade as a new provider result. Declare the native answer fields and validate their structural projection before accepting a scored artifact. Adapters must not remove returned candidates by URL meaning or consult expected answers when selecting candidate fields. Apply privacy redaction only to separately identified public views after scoring, preserving the private originals and distinguishing original, scored and public-view hashes. 5. Separate retrieval success, task acceptance, content quality, latency, cost, and freshness. Treat missing telemetry as unavailable. A request that received no first byte contributes to total latency, but not to first-byte or body-transfer stage means; show each stage’s observed population. A missing or partial reference makes the affected metric unavailable or scoped; it does not prove success or failure. Do not derive binary acceptance from an unrelated aggregate similarity score. diff --git a/AGENTS.md b/AGENTS.md index 0ebd81c78..910741603 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ These requirements apply to **all** report tracks; browser-specific rules below - For benchmark design, execution, scoring, result audit, report generation or release, read `.agents/skills/benchmark-report/SKILL.md`. Use `.agents/skills/benchmark-editorial/SKILL.md` additionally only for a new or substantially rewritten standalone interpretation article, official blog or public-account article. - Keep one report implementation in `harness/web_fetch/report.py`. The default `scripts/build_report.py` build uses persistent measured evidence (`~/.browser-eval/evidence/current/` by default, or `WEB_FETCH_EVIDENCE_DIR`) and writes `reports/webfetch/`; the independently bound WCXB build uses `--wcxb` and writes `reports/wcxb/`. Do not create versioned report implementations, compatibility renderers or redirect builds. - Run repository Python only through `make` targets or `uv run --frozen python`. `.python-version`, `uv.lock`, the Makefile and CI are the runtime contract; never fall back to system Python. -- Run every publishable provider cohort through `scripts/run_certified_benchmark.py`, which freezes executable task inputs. Existing compiled reference answers stay frozen; pre/post source checks require an explicit `source_verification.reference_mode: live_exact` task declaration. Missing newer acceptance-contract fields never imply a live-reference task. Rerunning an unchanged compiled dataset must not require historical source-qualification raw files or reopen cohort admission; source qualification belongs to adding tasks or changing references/acceptance rules. Direct `run_benchmark.py` output and incomplete cohorts are diagnostic evidence only; never assemble or publish them. +- Run every publishable provider cohort through `scripts/run_certified_benchmark.py`, which freezes executable task inputs. Existing compiled reference answers stay frozen; pre/post source checks require an explicit `source_verification.reference_mode: live_exact` or `live_body` task declaration. `live_body` qualifies current source text against unchanged frozen body requirements and never refreshes the reference. Missing newer acceptance-contract fields never imply a live-reference task. Rerunning an unchanged compiled dataset must not require historical source-qualification raw files or reopen cohort admission; source qualification belongs to adding tasks or changing references/acceptance rules. Direct `run_benchmark.py` output and incomplete cohorts are diagnostic evidence only; never assemble or publish them. - Select report instances explicitly with `--profile` (or an external `--authority-root` bundle for historical full-cohort formats). Store their matrices, publication targets and mutable version receipts outside Git alongside the frozen evidence. Each report and track owns its own publication binding; never borrow another report's target. - For the complete WebMainBench report, `evaluation/webmainbench-full/current-report.json` inside the selected external authority bundle is the immutable baseline pre-call profile bound by its cohort certificate. Independently certified variant cohorts are registered in `evaluation/webmainbench-full/`; discover the current variant registry and each variant's profile and certificate from that directory rather than hard-coding names. The method-binding file joins the baseline and all certified variant cohorts through locked profile hashes. Their `publication_state` records frozen pre-call state. The live site, share URL, and expected remote version are bound in `evaluation/webmainbench-full/publication-target.json` together with the baseline and all variant profile hashes; update this target after publication without changing or re-certifying historical calls. - For the current WCXB report, the source-qualified manifest is the entire evaluation population for quality, delivery, and latency. Keep source-invalid cases outside the current manifest, scored evidence, public package, and future provider calls; preserve earlier physical calls only in private historical evidence. The dataset page owns frozen task inputs, source URLs and human reference scope; it must not show measured browser scores. The home page owns aggregate comparisons, the result analysis owns the filterable method-case inventory, and case result pages own full responses. Keep a direct link from each dataset row to a safe frozen-page preview and its human reference. Default result filters must show a nonempty, clearly labeled diagnostic population when one exists, and filtered tables must shrink rather than leave a large blank viewport. diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 4f3a9e48d..b2edc0eb5 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -127,6 +127,17 @@ body regions, record contracts, composite fields and patterns as well as facts; a fact group is optional when another content requirement establishes the source scope. +For live URLs whose revised acceptance depends on frozen body text, explicitly +select `source_verification.reference_mode: live_body`. The existing certified +entry captures current source text before and after calls. Both observations +must satisfy the original body requirements; neither observation replaces the +frozen reference, regions or thresholds. The freezer retains a hash-bound +`source-qualification.txt` beside task sources. Preflight checks its freshness +and contents, and certification rechecks retained pre/post evidence. Missing or +incompatible sources reject the cohort; they are not provider failures. Small +changes within the declared coverage tolerance remain valid. Unsupported +consumers must reject the mode rather than omit its source checks. + ## WCXB scoring `wcxb_scoring.py` owns certified frozen-HTML scoring. The maintained command is @@ -1248,7 +1259,7 @@ Lightpanda 使用 `--dump markdown --json` 的原生正文;Obscura 使用其 不重新审核入选资格,不读取历史资格调查原文,也不因旧归档缺失而改成诊断结果。 新增题目、修改参考或验收规则时,仍须通过相应构建与修订流程核验来源。 固定答案不会因为有来源核验信息、旧核验日期或缺少新版验收契约而自动变成实时参考。 -只有题目明确声明 `source_verification.reference_mode: live_exact`,才执行调用前冻结和调用后核验。 +题目明确声明 `source_verification.reference_mode: live_exact` 时,执行调用前冻结和调用后精确核验;`live_body` 则在调用前后用原正文规则核验当前来源,保留原参考、分区和门槛。来源核验缺失或不通过时,不能认证为正式结果。 没有该声明时,执行器保持题库中的答案和评分规则不变。明确修订参考答案仍走独立的参考修订入口。 已授权的错误修正应自动完成:先查明共享原因并添加必要的正反测试,再冻结受影响的完整调用范围,经认证入口补跑。 diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index fa079cce6..9aa7043a7 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -31,6 +31,7 @@ ) from harness.web_fetch.reference_lifecycle import ( _completion, + body_source_qualifies, check_case_reference_binding, completed_case_records, legacy_reference_cases, @@ -253,6 +254,18 @@ def compute_reference_update(dataset, captures, observations=None, *, refresh_ca if missing_facts: raise ValueError('Required fact missing from independent reference: ' +cid+'/'+missing_facts[0]) + if case.get('source_verification', {}).get('reference_mode') == 'live_body': + if not body_source_qualifies(case, text): + raise ValueError('Current source no longer satisfies frozen body: ' + cid) + case['source_verification'].update( + state='valid', checked_at=timestamp, selector=selector, method=method, + observation_route='browser' if cid in observations else 'http', + reference_sha256=hashlib.sha256(case['ground_truth_main_content'].encode()).hexdigest(), + qualification_sha256=hashlib.sha256(text.encode()).hexdigest()) + contexts[cid] = {'row': row, 'missing_facts': missing_facts, + 'browser_observation': cid in observations, + 'qualification_text': text} + continue old=case['ground_truth_main_content'] if normalize_for_match(old)==normalize_for_match(text):text=old old_requirements=case.get('body_requirements') @@ -370,6 +383,8 @@ def main(argv=None): and not capture_record['quality_issues'] else 'needs_review') (target/'current-capture.json').write_text( json.dumps(capture_record,ensure_ascii=False,indent=2)+'\n') + if 'qualification_text' in contexts[cid]: + (target/'source-qualification.txt').write_text(contexts[cid]['qualification_text'], encoding='utf-8') (target/'reference.txt').write_text(case['ground_truth_main_content']) (args.output/'dataset.json').write_text(json.dumps(dataset,ensure_ascii=False,indent=2)+'\n') refresh={'schema_version':1,'changed':changes, @@ -391,7 +406,7 @@ def main(argv=None): json.dumps(dataset,ensure_ascii=False,indent=2)+'\n') (args.output/'reference-refresh.json').write_text( json.dumps(refresh,ensure_ascii=False,indent=2)+'\n') - print('Frozen exact-reference cases:',sum(requires_fresh_reference(c) for c in dataset['cases']),'; changed full-text references:', len(changes)) + print('Qualified live-reference cases:',sum(requires_fresh_reference(c) for c in dataset['cases']),'; changed full-text references:', len(changes)) def load_resumable_observations(checkpoint, cases): diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 00e4dc003..148c44080 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Any +from lexbench_webfetch.body_coverage import evaluate_body_requirements from lexbench_webfetch.metrics import normalize_for_match, normalize_resource_url from lexbench_webfetch.paths import contained_path, safe_identifier from lexbench_webfetch.scoring_regressions import valid_correction_provenance @@ -147,9 +148,9 @@ def requires_fresh_reference(case) -> bool: validate_acceptance_contract(contract) verification = _get(case, "source_verification", {}) or {} mode = verification.get("reference_mode", "frozen") - if mode not in {"frozen", "live_exact"}: - raise ReferenceLifecycleError("Unknown reference mode; use frozen or live_exact") - return mode == "live_exact" + if mode not in {"frozen", "live_exact", "live_body"}: + raise ReferenceLifecycleError("Unknown reference mode; use frozen, live_exact or live_body") + return mode in {"live_exact", "live_body"} def expected_providers(dataset, case): @@ -177,6 +178,29 @@ def reference_cases(dataset): return cases +def body_source_qualifies(case, content): + """Test an independent current body against unchanged declared body requirements.""" + body = _get(case, "body_requirements", {}) + if not body or not isinstance(content, str) or not content.strip(): + raise ReferenceLifecycleError("Live body qualification needs a body contract and source text") + return evaluate_body_requirements(content, body)["body_coverage_pass"] + + +def verify_body_qualification(case, source, verification): + """Recheck retained current-source bytes against the original body contract.""" + identifier = safe_identifier(_get(case, "id")) + try: + source = contained_path(input_tree(Path(source) / "dataset.json", "sources"), + f"{identifier}/source-qualification.txt", + reject_symlinks=True).read_text(encoding="utf-8") + except (OSError, ValueError) as error: + raise ReferenceLifecycleError(f"{identifier}: current body qualification is missing") from error + if sha256(source) != verification.get("qualification_sha256"): + raise ReferenceLifecycleError(f"{identifier}: current body qualification hash mismatch") + if not body_source_qualifies(case, source): + raise ReferenceLifecycleError(f"{identifier}: current source no longer satisfies frozen body", "reference_changed") + + def requires_browser_observation(case): """Select reference acquisition from declared provenance, never a task ID.""" verification = _get(case, 'source_verification', {}) or {} @@ -623,6 +647,8 @@ def _binding(case, dataset_path: Path, now: datetime) -> dict[str, Any]: actual_hash = sha256(reference) if verification.get("reference_sha256") != actual_hash: raise ReferenceLifecycleError(f"{identifier}: reference SHA-256 mismatch", "reference_changed") + if verification.get("reference_mode") == "live_body": + verify_body_qualification(case, dataset_path.parent, verification) binding = { "reference_sha256": actual_hash, "expected_normalized_sha256": sha256(normalize_for_match(reference)), @@ -908,7 +934,10 @@ def _postcheck_case(binding, case, observation, records, now, original_case=None normalized = sha256(normalize_for_match(content)) if "content_sha256" in observation and observation["content_sha256"] != actual: raise ReferenceLifecycleError("Observation content hash mismatch") - return {"status": "verified" if normalized == binding["expected_normalized_sha256"] else "reference_changed", + qualified = (body_source_qualifies(case, content) + if binding["source_verification"].get("reference_mode") == "live_body" + else normalized == binding["expected_normalized_sha256"]) + return {"status": "verified" if qualified else "reference_changed", "checked_at": observation["checked_at"], "latest_call_finished_at": latest.isoformat(), "content_sha256": actual, "normalized_sha256": normalized} @@ -1708,11 +1737,20 @@ def valid_hash(value): for identifier, binding in bindings.items(): safe_identifier(identifier) result = post["cases"][identifier] + live_body = binding.get("source_verification", {}).get("reference_mode") == "live_body" + if live_body: + if source is None or observations is None: + raise ReferenceLifecycleError("Live body certificate requires retained source observations") + original_body_case = {'id': identifier, 'body_requirements': + binding.get('scope_identity', {}).get('evaluation_contract', {}).get('body_requirements')} + verify_body_qualification(original_body_case, source, binding['source_verification']) + if not body_source_qualifies(original_body_case, observations[identifier].get('content_text')): + raise ReferenceLifecycleError("Post-call source no longer satisfies frozen body", "reference_changed") if (not isinstance(result, dict) or result.get("status") != "verified" or not valid_hash(binding.get("reference_sha256")) or not valid_hash(binding.get("expected_normalized_sha256")) or not valid_hash(result.get("content_sha256")) - or result.get("normalized_sha256") != binding["expected_normalized_sha256"] + or (not live_body and result.get("normalized_sha256") != binding["expected_normalized_sha256"]) or not isinstance(binding.get("scope_identity"), dict) or binding["scope_identity"].get("case_id") != identifier or not isinstance(binding["scope_identity"].get("evaluation_contract"), dict)): diff --git a/tests/test_live_body_qualification.py b/tests/test_live_body_qualification.py new file mode 100644 index 000000000..6b6603be6 --- /dev/null +++ b/tests/test_live_body_qualification.py @@ -0,0 +1,262 @@ +"""Current-source qualification preserves frozen body references and tolerances.""" + +import json +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from lexbench_webfetch.body_coverage import body_requirements + +from harness.web_fetch import reference_freeze +from harness.web_fetch.reference_lifecycle import ( + ReferenceLifecycleError, + _postcheck_case, + preflight_reference_checks, + reference_cases, + sha256, +) + +TEXT = "Birds navigate by observing the sun and stars while migrating over long distances. " * 12 +NOW = datetime.now(UTC) + + +def inputs(tmp_path, current=TEXT): + body = body_requirements(TEXT, structure_hint="whole_declared_component") + body.update(minimum_coverage=0.8, minimum_floor=0.8, minimum_region_passes=len(body["regions"])) + case = { + "id": "body", + "name": "Body", + "url": "https://example.test/body", + "track": "web", + "task_kind": "body", + "task_prompt": "Read the full article", + "expected_any": [], + "min_content_chars": 1, + "ground_truth_main_content": TEXT, + "body_requirements": body, + "source_verification": {"reference_mode": "live_body"}, + } + dataset = {"cases": [case], "coverage_plan": {"web": ["fixture"]}} + captures = tmp_path / "captures" + captures.mkdir() + folder = captures / "body" + folder.mkdir() + (folder / "reference.txt").write_text(current) + (folder / "source.html").write_text(current) + (captures / "audit.json").write_text( + json.dumps( + [ + { + "case_id": "body", + "status": 200, + "state": "captured", + "checked_at": NOW.isoformat(), + "selector": "main", + "sha256": sha256(current), + "extracted_sha256": sha256(current), + "quality_issues": [], + } + ] + ) + ) + source = tmp_path / "dataset.json" + source.write_text(json.dumps(dataset)) + return dataset, captures, source + + +@pytest.mark.parametrize("changed", [False, True]) +def test_freezer_keeps_original_body_and_reference_with_a_current_receipt(tmp_path, changed): + current = TEXT.replace("stars", "moon") if changed else TEXT + dataset, captures, source = inputs(tmp_path, current) + old = deepcopy(dataset) + output = tmp_path / "frozen" + reference_freeze.main( + ["--dataset", str(source), "--captures", str(captures), "--output", str(output)] + ) + frozen = json.loads((output / "dataset.json").read_text()) + case = frozen["cases"][0] + assert case["ground_truth_main_content"] == TEXT + assert case["body_requirements"] == old["cases"][0]["body_requirements"] + assert dataset == old + assert set(reference_cases(frozen)) == {"body"} + assert (output / "sources/body/source-qualification.txt").read_text() == current + checks = preflight_reference_checks(frozen["cases"], output / "dataset.json", now=NOW) + assert set(checks["cases"]) == {"body"} + assert not json.loads((output / "reference-refresh.json").read_text())["reference_changed"] + + +@pytest.mark.parametrize("current", ["Unrelated rewritten official article. " * 80, ""]) +def test_changed_or_missing_source_is_rejected_before_scoring(tmp_path, current): + dataset, captures, _ = inputs(tmp_path, current) + with pytest.raises(ValueError, match="source|qualification"): + reference_freeze.compute_reference_update(dataset, captures) + + +@pytest.mark.parametrize("damage", ["missing", "tampered", "stale"]) +def test_preflight_requires_intact_fresh_qualification(tmp_path, damage): + dataset, captures, source = inputs(tmp_path) + output = tmp_path / "frozen" + reference_freeze.main( + ["--dataset", str(source), "--captures", str(captures), "--output", str(output)] + ) + frozen = json.loads((output / "dataset.json").read_text()) + receipt = output / "sources/body/source-qualification.txt" + if damage == "missing": + receipt.unlink() + if damage == "tampered": + receipt.write_text("changed") + if damage == "stale": + frozen["cases"][0]["source_verification"]["checked_at"] = "2020-01-01T00:00:00Z" + with pytest.raises(ReferenceLifecycleError): + preflight_reference_checks(frozen["cases"], output / "dataset.json", now=NOW) + + +@pytest.mark.parametrize( + "post,expected", + [ + (TEXT, "verified"), + (TEXT.replace("stars", "moon"), "verified"), + ("A different article. " * 80, "reference_changed"), + ], +) +def test_postcheck_uses_frozen_body_tolerance_not_exact_current_text(tmp_path, post, expected): + dataset, captures, source = inputs(tmp_path) + output = tmp_path / "frozen" + reference_freeze.main( + ["--dataset", str(source), "--captures", str(captures), "--output", str(output)] + ) + frozen = json.loads((output / "dataset.json").read_text()) + case = frozen["cases"][0] + binding = preflight_reference_checks([case], output / "dataset.json", now=NOW)["cases"]["body"] + records = [ + { + "started_at": NOW.isoformat(), + "elapsed_ms": 1000, + "metrics": {"reference_sha256": sha256(TEXT)}, + } + ] + observation = { + "independent": True, + "scope_identity": binding["scope_identity"], + "completion": {"satisfied": True, "elapsed_ms": 1, "timeout_ms": 1000}, + "checked_at": (NOW + timedelta(seconds=2)).isoformat(), + "content_text": post, + } + result = _postcheck_case(binding, case, observation, records, NOW + timedelta(seconds=3)) + assert result["status"] == expected + + +@pytest.mark.parametrize("invalid_phase", [None, "pre", "post"]) +def test_formal_entry_captures_before_calls_and_rejects_invalid_source( + tmp_path, monkeypatch, invalid_phase +): + from harness.web_fetch import benchmark_runner, certification, evidence + from harness.web_fetch.reference_lifecycle import ( + require_verified_reference_checks, + scope_identity, + ) + + current = ( + "A completely rewritten document about unrelated subjects. " * 50 + if invalid_phase == "pre" + else TEXT.replace("stars", "moon") + ) + dataset, _, source = inputs(tmp_path, current) + dataset["cases"][0]["category"] = "web" + dataset["tool"] = "web_fetch" + dataset["comparison"] = {} + source.write_text(json.dumps(dataset)) + output = tmp_path / "run" + work = tmp_path / "work" + config = tmp_path / "providers.json" + config.write_text(json.dumps({"providers": {"fixture": {"type": "fixture"}}})) + stages = [] + + def capture(destination, case=None): + case = case or dataset["cases"][0] + observed = ( + "A different current document. " * 50 + if invalid_phase == "post" and destination.name == "post-captures" + else current + ) + folder = destination / "body" + folder.mkdir(parents=True, exist_ok=True) + (folder / "reference.txt").write_text(observed) + (folder / "source.html").write_text(observed) + row = { + "case_id": "body", + "url": case["url"], + "status": 200, + "state": "captured", + "missing_facts": [], + "quality_issues": [], + "selector": "main", + "checked_at": datetime.now(UTC).isoformat(), + "sha256": sha256(observed), + "reference_sha256": sha256(observed), + "extracted_sha256": sha256(observed), + "scope_identity": scope_identity(case), + "independent": True, + "source": "independent HTTP GET (no provider output)", + "completion": {"satisfied": True, "elapsed_ms": 1, "timeout_ms": 1000}, + } + (destination / "audit.json").write_text(json.dumps([row])) + (folder / "evidence.json").write_text(json.dumps(row)) + + def execute(stage, arguments): + name = stage.__module__.rsplit(".", 1)[-1] + "." + stage.__name__ + stages.append(name) + if name == "source_capture.main": + selected = json.loads(Path(arguments[0]).read_text())["cases"][0] + capture(Path(arguments[1]), selected) + elif name == "reference_freeze.main": + stage(arguments) + elif name == "benchmark_runner.main": + monkeypatch.setattr(benchmark_runner, "load_runtime_env", lambda *a, **k: None) + stage(arguments) + elif name == "reference_freeze.postcheck_main": + monkeypatch.setattr( + reference_freeze, "capture", lambda case, destination: capture(destination, case) + ) + stage(arguments) + elif name == "evidence.finalize_main": + evidence.finalize_aggregate(output, work / "post-captures/observations.json") + else: + pytest.fail("Unexpected stage " + name) + + monkeypatch.setattr(certification, "run", execute) + args = [ + "--dataset", + str(source), + "--output", + str(output), + "--work-root", + str(work), + "--providers", + "fixture", + "--provider-config", + str(config), + "--case-ids", + "body", + ] + if invalid_phase: + with pytest.raises((SystemExit, ValueError)): + certification.main(args) + if invalid_phase == "pre": + assert "benchmark_runner.main" not in stages + assert not output.exists() + else: + assert "reference_freeze.postcheck_main" in stages + else: + certification.main(args) + assert stages[0] == "source_capture.main" + assert "reference_freeze.postcheck_main" in stages + saved = json.loads((output / "dataset.json").read_text()) + assert saved["cases"][0]["ground_truth_main_content"] == TEXT + manifest = json.loads((output / "execution.json").read_text())["batches"] + records = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] + require_verified_reference_checks(manifest, cases=saved, records=records, source=output) + (output / "sources/body/source-qualification.txt").unlink() + with pytest.raises(ReferenceLifecycleError, match="qualification is missing"): + require_verified_reference_checks(manifest, cases=saved, records=records, source=output) From 8a68908cc1bddb62b5624b01d3192e3a0f95b709 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:30:27 +0800 Subject: [PATCH 2/3] fix(webfetch): retain separate qualification receipts across batches --- harness/web_fetch/README.md | 2 +- harness/web_fetch/artifacts.py | 39 ++++++++----- harness/web_fetch/evidence.py | 2 +- harness/web_fetch/reference_freeze.py | 3 +- harness/web_fetch/reference_lifecycle.py | 9 ++- harness/web_fetch/reference_revision.py | 1 + harness/web_fetch/refresh_assembly.py | 2 + tests/test_live_body_qualification.py | 72 +++++++++++++++++++++--- 8 files changed, 105 insertions(+), 25 deletions(-) diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index b2edc0eb5..a9fafe118 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -132,7 +132,7 @@ select `source_verification.reference_mode: live_body`. The existing certified entry captures current source text before and after calls. Both observations must satisfy the original body requirements; neither observation replaces the frozen reference, regions or thresholds. The freezer retains a hash-bound -`source-qualification.txt` beside task sources. Preflight checks its freshness +`source-qualification-.txt` beside task sources. Preflight checks its freshness and contents, and certification rechecks retained pre/post evidence. Missing or incompatible sources reject the cohort; they are not provider failures. Small changes within the declared coverage tolerance remain valid. Unsupported diff --git a/harness/web_fetch/artifacts.py b/harness/web_fetch/artifacts.py index 7b968cd10..8bbbdeba2 100644 --- a/harness/web_fetch/artifacts.py +++ b/harness/web_fetch/artifacts.py @@ -18,7 +18,9 @@ from harness.record_io import file_sha256 from harness.web_fetch import native_routes from harness.web_fetch.publication_privacy import original_evidence_size, read_evidence_binding +from harness.web_fetch.reference_inputs import input_tree from harness.web_fetch.reference_lifecycle import ( + body_qualification_path, latency_correction_original_elapsed, preserve_legacy_reference_dataset, ) @@ -408,7 +410,8 @@ def materialize_bound_artifacts(output: Path, source_root: Path, records: list[d verify_record_public_call_logs(output, record) -def materialize_reference_checks(output: Path, source: Path, batches: list[dict]) -> None: +def materialize_reference_checks(output: Path, source: Path, batches: list[dict], *, + qualification_source: Path | None = None) -> None: """Retain hash-bound source observations and native execution contracts.""" native_routes.materialize_native_route_contracts(output, source, batches) for batch in batches: @@ -421,18 +424,28 @@ def materialize_reference_checks(output: Path, source: Path, batches: list[dict] or any(char not in '0123456789abcdef' for char in digest) or relative != f'reference-checks/observations-{digest}.json'): raise ValueError('Reference check has an invalid evidence binding') - path = contained_path(source, relative, reject_symlinks=True) - if not path.is_file() or file_sha256(path) != digest: - raise ValueError('Reference check evidence differs from its certificate') - target = contained_path(output, relative, reject_symlinks=True) - if target.exists(): - if not target.is_file() or file_sha256(target) != digest: - raise ValueError('Conflicting reference check evidence') - else: - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, target) - if file_sha256(target) != digest: - raise ValueError('Copied reference check evidence changed') + files = [(relative, digest)] + for identifier, binding in batch.get('reference_checks', {}).get('cases', {}).items(): + verification = binding.get('source_verification', {}) + if verification.get('reference_mode') == 'live_body': + digest = verification.get('qualification_sha256') + files.append(('sources/' + body_qualification_path(identifier, digest), digest)) + for relative, digest in files: + path = (contained_path(input_tree((qualification_source or source) / 'dataset.json', 'sources'), + relative.removeprefix('sources/'), reject_symlinks=True) + if relative.startswith('sources/') + else contained_path(source, relative, reject_symlinks=True)) + if not path.is_file() or file_sha256(path) != digest: + raise ValueError('Reference check evidence differs from its certificate') + target = contained_path(output, relative, reject_symlinks=True) + if target.exists(): + if not target.is_file() or file_sha256(target) != digest: + raise ValueError('Conflicting reference check evidence') + else: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + if file_sha256(target) != digest: + raise ValueError('Copied reference check evidence changed') def materialize_task_references(output: Path, trees: dict[str, Path], case_ids: Iterable[str]) -> None: diff --git a/harness/web_fetch/evidence.py b/harness/web_fetch/evidence.py index 24a3dcce4..029a7ea46 100644 --- a/harness/web_fetch/evidence.py +++ b/harness/web_fetch/evidence.py @@ -726,7 +726,7 @@ def finalize_aggregate(run_dir: Path, observations: Path) -> None: raise ValueError("Finalized worker metadata changed") for path, manifest in zip(manifests, updated, strict=True): - materialize_reference_checks(run_dir, path.parent, [manifest]) + materialize_reference_checks(run_dir, path.parent, [manifest], qualification_source=run_dir) rebuilt = {**execution, "batches": [updated_by_key[_attempt_key(batch)] for batch in original]} temporary = None diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index 9aa7043a7..9d68f3a35 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -31,6 +31,7 @@ ) from harness.web_fetch.reference_lifecycle import ( _completion, + body_qualification_path, body_source_qualifies, check_case_reference_binding, completed_case_records, @@ -384,7 +385,7 @@ def main(argv=None): (target/'current-capture.json').write_text( json.dumps(capture_record,ensure_ascii=False,indent=2)+'\n') if 'qualification_text' in contexts[cid]: - (target/'source-qualification.txt').write_text(contexts[cid]['qualification_text'], encoding='utf-8') + (args.output/'sources'/body_qualification_path(cid, case['source_verification']['qualification_sha256'])).write_text(contexts[cid]['qualification_text'], encoding='utf-8') (target/'reference.txt').write_text(case['ground_truth_main_content']) (args.output/'dataset.json').write_text(json.dumps(dataset,ensure_ascii=False,indent=2)+'\n') refresh={'schema_version':1,'changed':changes, diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 148c44080..8dd4a1673 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -186,12 +186,19 @@ def body_source_qualifies(case, content): return evaluate_body_requirements(content, body)["body_coverage_pass"] +def body_qualification_path(identifier, digest): + """Retain distinct current captures of the same unchanged reference.""" + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ReferenceLifecycleError("Invalid body qualification hash") + return f"{safe_identifier(identifier)}/source-qualification-{digest}.txt" + + def verify_body_qualification(case, source, verification): """Recheck retained current-source bytes against the original body contract.""" identifier = safe_identifier(_get(case, "id")) try: source = contained_path(input_tree(Path(source) / "dataset.json", "sources"), - f"{identifier}/source-qualification.txt", + body_qualification_path(identifier, verification.get("qualification_sha256")), reject_symlinks=True).read_text(encoding="utf-8") except (OSError, ValueError) as error: raise ReferenceLifecycleError(f"{identifier}: current body qualification is missing") from error diff --git a/harness/web_fetch/reference_revision.py b/harness/web_fetch/reference_revision.py index 1a404beac..296b2677b 100644 --- a/harness/web_fetch/reference_revision.py +++ b/harness/web_fetch/reference_revision.py @@ -62,6 +62,7 @@ "checked_at", "http_status", "source_snapshot_sha256", + "qualification_sha256", "extracted_reference_sha256", "browser_observation_sha256", } diff --git a/harness/web_fetch/refresh_assembly.py b/harness/web_fetch/refresh_assembly.py index 4b415ca66..c4ec21e4e 100644 --- a/harness/web_fetch/refresh_assembly.py +++ b/harness/web_fetch/refresh_assembly.py @@ -869,6 +869,7 @@ def _require_canonical_cases( for field in ( "reference_sha256", "source_snapshot_sha256", + "qualification_sha256", "extracted_reference_sha256", "browser_observation_sha256", ): @@ -881,6 +882,7 @@ def _require_canonical_cases( "checked_at", "http_status", "source_snapshot_sha256", + "qualification_sha256", "extracted_reference_sha256", "browser_observation_sha256", ): diff --git a/tests/test_live_body_qualification.py b/tests/test_live_body_qualification.py index 6b6603be6..bff7b3e7a 100644 --- a/tests/test_live_body_qualification.py +++ b/tests/test_live_body_qualification.py @@ -80,7 +80,7 @@ def test_freezer_keeps_original_body_and_reference_with_a_current_receipt(tmp_pa assert case["body_requirements"] == old["cases"][0]["body_requirements"] assert dataset == old assert set(reference_cases(frozen)) == {"body"} - assert (output / "sources/body/source-qualification.txt").read_text() == current + assert next((output / "sources/body").glob("source-qualification-*.txt")).read_text() == current checks = preflight_reference_checks(frozen["cases"], output / "dataset.json", now=NOW) assert set(checks["cases"]) == {"body"} assert not json.loads((output / "reference-refresh.json").read_text())["reference_changed"] @@ -101,7 +101,7 @@ def test_preflight_requires_intact_fresh_qualification(tmp_path, damage): ["--dataset", str(source), "--captures", str(captures), "--output", str(output)] ) frozen = json.loads((output / "dataset.json").read_text()) - receipt = output / "sources/body/source-qualification.txt" + receipt = next((output / "sources/body").glob("source-qualification-*.txt")) if damage == "missing": receipt.unlink() if damage == "tampered": @@ -147,10 +147,7 @@ def test_postcheck_uses_frozen_body_tolerance_not_exact_current_text(tmp_path, p assert result["status"] == expected -@pytest.mark.parametrize("invalid_phase", [None, "pre", "post"]) -def test_formal_entry_captures_before_calls_and_rejects_invalid_source( - tmp_path, monkeypatch, invalid_phase -): +def certified_run(tmp_path, monkeypatch, invalid_phase=None, current_text=None): from harness.web_fetch import benchmark_runner, certification, evidence from harness.web_fetch.reference_lifecycle import ( require_verified_reference_checks, @@ -160,7 +157,7 @@ def test_formal_entry_captures_before_calls_and_rejects_invalid_source( current = ( "A completely rewritten document about unrelated subjects. " * 50 if invalid_phase == "pre" - else TEXT.replace("stars", "moon") + else current_text or TEXT.replace("stars", "moon") ) dataset, _, source = inputs(tmp_path, current) dataset["cases"][0]["category"] = "web" @@ -257,6 +254,65 @@ def execute(stage, arguments): manifest = json.loads((output / "execution.json").read_text())["batches"] records = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] require_verified_reference_checks(manifest, cases=saved, records=records, source=output) - (output / "sources/body/source-qualification.txt").unlink() + receipt = next((output / "sources/body").glob("source-qualification-*.txt")) + original = receipt.read_bytes() + receipt.unlink() with pytest.raises(ReferenceLifecycleError, match="qualification is missing"): require_verified_reference_checks(manifest, cases=saved, records=records, source=output) + + receipt.write_bytes(original) + return output + + +@pytest.mark.parametrize("invalid_phase", [None, "pre", "post"]) +def test_formal_entry_captures_before_calls_and_rejects_invalid_source( + tmp_path, monkeypatch, invalid_phase +): + certified_run(tmp_path, monkeypatch, invalid_phase) + + +def test_combined_batches_retain_distinct_qualified_sources(tmp_path, monkeypatch): + from harness.web_fetch.artifacts import materialize_reference_checks + from harness.web_fetch.reference_lifecycle import require_verified_reference_checks + from harness.web_fetch.refresh_assembly import _require_canonical_cases + + combined = tmp_path / "combined" + combined.mkdir() + runs = [] + for word in ("moon", "clouds"): + root = tmp_path / word + root.mkdir() + runs.append(certified_run(root, monkeypatch, current_text=TEXT.replace("stars", word))) + canonical = json.loads((runs[0] / "dataset.json").read_text()) + (combined / "dataset.json").write_text(json.dumps(canonical)) + canonical_cases = {case["id"]: case for case in canonical["cases"]} + for run in runs: + supplied = json.loads((run / "dataset.json").read_text()) + batches = json.loads((run / "execution.json").read_text())["batches"] + rows = [json.loads(line) for line in (run / "results.jsonl").read_text().splitlines()] + _require_canonical_cases( + supplied, canonical_cases, rows, "batch", equivalent_capture_bindings=True + ) + materialize_reference_checks(combined, run, batches) + require_verified_reference_checks( + batches, + cases=canonical, + records=rows, + source=combined, + equivalent_capture_bindings=True, + ) + changed = deepcopy(supplied) + changed["cases"][0]["body_requirements"]["minimum_coverage"] = 0.5 + with pytest.raises(ValueError, match="differs from canonical"): + _require_canonical_cases( + changed, canonical_cases, rows, "batch", equivalent_capture_bindings=True + ) + receipts = list((combined / "sources/body").glob("source-qualification-*.txt")) + assert len(receipts) == 2 + assert {path.read_text() for path in receipts} == { + TEXT.replace("stars", word) for word in ("moon", "clouds") + } + receipt = next((runs[1] / "sources/body").glob("source-qualification-*.txt")) + receipt.write_text("tampered") + with pytest.raises(ValueError, match="differs from its certificate"): + materialize_reference_checks(combined, runs[1], batches) From d6f366597e7ca88da2bb534151294fa01e5f32f6 Mon Sep 17 00:00:00 2001 From: Maisiqi <324751762+Maisiqi@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:41:08 +0800 Subject: [PATCH 3/3] fix(webfetch): verify qualification receipts as raw bytes --- harness/web_fetch/reference_freeze.py | 2 +- harness/web_fetch/reference_lifecycle.py | 4 ++-- tests/test_live_body_qualification.py | 13 +++++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index 9d68f3a35..2293c88fd 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -385,7 +385,7 @@ def main(argv=None): (target/'current-capture.json').write_text( json.dumps(capture_record,ensure_ascii=False,indent=2)+'\n') if 'qualification_text' in contexts[cid]: - (args.output/'sources'/body_qualification_path(cid, case['source_verification']['qualification_sha256'])).write_text(contexts[cid]['qualification_text'], encoding='utf-8') + (args.output/'sources'/body_qualification_path(cid, case['source_verification']['qualification_sha256'])).write_bytes(contexts[cid]['qualification_text'].encode('utf-8')) (target/'reference.txt').write_text(case['ground_truth_main_content']) (args.output/'dataset.json').write_text(json.dumps(dataset,ensure_ascii=False,indent=2)+'\n') refresh={'schema_version':1,'changed':changes, diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 8dd4a1673..c4f38bf30 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -199,12 +199,12 @@ def verify_body_qualification(case, source, verification): try: source = contained_path(input_tree(Path(source) / "dataset.json", "sources"), body_qualification_path(identifier, verification.get("qualification_sha256")), - reject_symlinks=True).read_text(encoding="utf-8") + reject_symlinks=True).read_bytes() except (OSError, ValueError) as error: raise ReferenceLifecycleError(f"{identifier}: current body qualification is missing") from error if sha256(source) != verification.get("qualification_sha256"): raise ReferenceLifecycleError(f"{identifier}: current body qualification hash mismatch") - if not body_source_qualifies(case, source): + if not body_source_qualifies(case, source.decode("utf-8")): raise ReferenceLifecycleError(f"{identifier}: current source no longer satisfies frozen body", "reference_changed") diff --git a/tests/test_live_body_qualification.py b/tests/test_live_body_qualification.py index bff7b3e7a..9065f354b 100644 --- a/tests/test_live_body_qualification.py +++ b/tests/test_live_body_qualification.py @@ -93,9 +93,9 @@ def test_changed_or_missing_source_is_rejected_before_scoring(tmp_path, current) reference_freeze.compute_reference_update(dataset, captures) -@pytest.mark.parametrize("damage", ["missing", "tampered", "stale"]) +@pytest.mark.parametrize("damage", ["missing", "tampered", "newlines", "stale"]) def test_preflight_requires_intact_fresh_qualification(tmp_path, damage): - dataset, captures, source = inputs(tmp_path) + dataset, captures, source = inputs(tmp_path, TEXT + "\n") output = tmp_path / "frozen" reference_freeze.main( ["--dataset", str(source), "--captures", str(captures), "--output", str(output)] @@ -106,6 +106,8 @@ def test_preflight_requires_intact_fresh_qualification(tmp_path, damage): receipt.unlink() if damage == "tampered": receipt.write_text("changed") + if damage == "newlines": + receipt.write_bytes(receipt.read_bytes().replace(b"\n", b"\r\n")) if damage == "stale": frozen["cases"][0]["source_verification"]["checked_at"] = "2020-01-01T00:00:00Z" with pytest.raises(ReferenceLifecycleError): @@ -157,7 +159,7 @@ def certified_run(tmp_path, monkeypatch, invalid_phase=None, current_text=None): current = ( "A completely rewritten document about unrelated subjects. " * 50 if invalid_phase == "pre" - else current_text or TEXT.replace("stars", "moon") + else (current_text or TEXT.replace("stars", "moon")) + "\n" ) dataset, _, source = inputs(tmp_path, current) dataset["cases"][0]["category"] = "web" @@ -256,6 +258,9 @@ def execute(stage, arguments): require_verified_reference_checks(manifest, cases=saved, records=records, source=output) receipt = next((output / "sources/body").glob("source-qualification-*.txt")) original = receipt.read_bytes() + receipt.write_bytes(original.replace(b"\n", b"\r\n")) + with pytest.raises(ReferenceLifecycleError, match="qualification hash mismatch"): + require_verified_reference_checks(manifest, cases=saved, records=records, source=output) receipt.unlink() with pytest.raises(ReferenceLifecycleError, match="qualification is missing"): require_verified_reference_checks(manifest, cases=saved, records=records, source=output) @@ -310,7 +315,7 @@ def test_combined_batches_retain_distinct_qualified_sources(tmp_path, monkeypatc receipts = list((combined / "sources/body").glob("source-qualification-*.txt")) assert len(receipts) == 2 assert {path.read_text() for path in receipts} == { - TEXT.replace("stars", word) for word in ("moon", "clouds") + TEXT.replace("stars", word) + "\n" for word in ("moon", "clouds") } receipt = next((runs[1] / "sources/body").glob("source-qualification-*.txt")) receipt.write_text("tampered")