From b3b6f72f1b0de51707646b7ceb8666f0835fd213 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 10:41:59 +0530 Subject: [PATCH 01/40] docs: design FreshData TruthBench --- .../2026-07-14-freshdata-truthbench-design.md | 499 ++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md diff --git a/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md b/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md new file mode 100644 index 0000000..3cce5ba --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md @@ -0,0 +1,499 @@ +# FreshData TruthBench Design + +**Date:** 2026-07-14 +**Target:** `origin/main` at `a1da862` +**Implementation branch:** `feature/truthbench-jwd` +**Status:** Approved for implementation planning + +## Purpose + +FreshData TruthBench is a deterministic semantic red-team and regression system. +It continuously searches for cases where a public FreshData data-quality surface +makes an unsafe, incorrect, inconsistent, or unexplained decision, then converts +each reproduced failure into a minimized case and a permanent regression test. + +TruthBench is test infrastructure, not a cleaning engine. No LLM, teacher model, +provider hook, or generative runtime may participate in FreshData's default +cleaning path or in a TruthBench release run. An LLM may only help humans author +adversarial seed cases, criticize results, and analyze reproduced failures outside +the executable benchmark. + +## Repository Context + +TruthBench is additive to the existing quality layers: + +- CleanBench measures frame-level repair fidelity, calibration, preservation, + profile replay, privacy, and performance. +- The Validation Gauntlet measures labelled dispositions across five fixtures, + but uses partial cell labels and a deliberately loose value comparator. +- The enterprise fixture benchmark measures repair, preservation, trust + monotonicity, reporting completeness, and scale. +- Golden, real-world, online, domain, integration, backend, Copilot, privacy, and + generated-code pytest suites protect individual public contracts. +- The performance investigation provides subprocess-isolated scalability + evidence and is not a semantic correctness oracle. + +TruthBench does not replace, rename, weaken, or rebaseline these systems. It adds +the strict cross-surface cell-level contract they do not currently provide. + +The isolated baseline run on the target commit produced 3,692 passes, 11 skips, +12 deselections, and 93.66% coverage. Two existing throughput assertions failed +during the seven-minute full run: +`balanced-abalone` missed its allowed floor by approximately 0.004%, and +`aggressive-gapminder` was slower than its allowed floor. Both exact tests passed +immediately when rerun with coverage enabled. These are recorded as pre-existing, +non-reproducible environment timing flakes. TruthBench must not relax, skip, or +change either expectation. + +## Chosen Approach + +TruthBench will live in a dedicated `benchmarks/truthbench/` package. It may reuse +stable public APIs and small general-purpose helpers from existing benchmarks, +but its oracle, exact comparator, result schema, gates, surface adapters, failure +reducer, and eight-domain corpus are independent. + +This is preferred over extending the Validation Gauntlet because the Gauntlet's +loose canonical equality and partial labels are part of its historical contract. +It is preferred over a CleanBench T6 track because CleanBench is organized around +aggregate frame metrics rather than one record per cell and surface. + +## Scope + +### In scope + +TruthBench will: + +1. Generate deterministic gold-labelled fixtures for finance, healthcare, + retail, CRM, logistics, government, education, and insurance. +2. Assign exactly one disposition to every test cell: + `preserve`, `repair`, `flag`, or `review`. +3. Exercise every public FreshData surface that makes, applies, transports, + renders, serializes, or gates a data-quality decision. +4. Normalize heterogeneous surface outputs into a common decision record. +5. Compare actual decision, expected decision, output value, confidence, + rationale, audit completeness, trust-score change, backend consistency, and + determinism. +6. Detect raw PII leakage into reports, logs, model context, generated code, + rendered output, CLI output, and committed benchmark artifacts. +7. Minimize every reproduced failure to a stable, privacy-safe case. +8. Enforce absolute PR and release gates independent of baseline-relative trends. +9. Commit versioned machine-readable results and a human-readable report. + +### Out of scope + +TruthBench will not: + +- add an LLM or network call to `fd.clean`, semantic routing, domain repair, + privacy processing, trust scoring, or CI; +- automatically approve or apply an ambiguous repair; +- silently accept documented backend divergence; +- use timing, timestamps, process memory, random salts, or generated identifiers + as decision-determinism inputs; +- replace performance benchmarks with correctness measurements; +- treat an unavailable required backend as a skip; +- overwrite benchmark gold labels to make an implementation pass. + +## Disposition Contract + +Each physical test cell has a stable identifier: +`:::`. + +The four dispositions mean: + +| Disposition | Required default behavior | +|---|---| +| `preserve` | The value is valid, possibly unusual. Mutating its value or semantic meaning is corruption. An error-severity false alarm is also a failure. | +| `repair` | A safe deterministic repair has one gold output. A mutating surface must produce that exact value and dtype; a read-only surface must identify the problem without inventing another value. | +| `flag` | The value must be surfaced with evidence but remain unchanged by default. | +| `review` | The value is ambiguous or policy-conflicted and must enter a human-review, quarantine, reject, or explicit suggestion path without an automatic guess. | + +All pristine background cells default to `preserve`. Adversarial injections replace +that label explicitly. Fixture construction fails if any cell has no label, more +than one label, a missing stable ID, or an invalid expected output. + +Row-level expectations such as exact duplicate removal and schema-level +expectations such as added, missing, renamed, or type-drifted columns are stored as +separate case records. They never substitute for cell labels. + +## Gold Fixture Design + +Each domain fixture contains a pristine frame, an adversarial frame, a complete +cell-label matrix, explicit schemas, context policies, protected columns, PII +canaries, and row/schema expectations. Generation is pure Python with fixed seeds, +fixed reference dates, fixed timezone, fixed locale assumptions, and stable row +identifiers. + +The corpus includes: + +- valid extremes, rare categories, leading-zero identifiers, Unicode names, + uncommon but valid codes, zero-value transactions, and empty free text; +- ambiguous decimal separators, numeric dates, percentages, abbreviated units, + policy aliases, category variants, and case-sensitive identifiers; +- mixed kilograms/pounds, Celsius/Fahrenheit, percentages/fractions, local time + zones, currencies, currency symbols, and Indian/international number grouping; +- English and non-English text, mixed scripts, emoji, combining characters, + mojibake, zero-width characters, HTML entities, and alternate encodings; +- ISO, US, European, textual, partial, timezone-aware, and impossible dates; +- hidden email, phone, SSN/national-ID, card, address, medical, and FERPA-style + identifiers in free text, numeric columns, category values, and late rows; +- added, removed, reordered, renamed, duplicated, and type-drifted columns; +- contradictory protection/repair, currency/unit, range, locale, and mapping + policies; +- semantic traps where `apple`, `Apple`, and `AAPL` have different correct + meanings based on column, schema, and domain. + +Domain emphasis: + +- **Finance:** instruments, company names, tickers, prices, currencies, ledger + signs, percentages, settlement dates, and transaction memos. +- **Healthcare:** patient identifiers, clinical codes, lab values and units, + dates of birth, vital signs, free-text notes, and protected health information. +- **Retail:** SKUs, quantities, prices, promotions, returns, product names, + currencies, reviews, and zero-price edge cases. +- **CRM:** customer IDs, Unicode names, email, phone, country codes, lifecycle + states, signup dates, and free-text contact notes. +- **Logistics:** shipment IDs, UN/LOCODE-like locations, weights, dimensions, + temperatures, time zones, tracking states, delivery windows, and addresses. +- **Government:** case IDs, agency and district codes, postal identifiers, + multilingual values, fiscal amounts, public dates, and restricted identifiers. +- **Education:** student IDs, grades, assessment scales, school years, programs, + enrollment dates, guardian details, and FERPA-protected notes. +- **Insurance:** claim and policy IDs, coverage codes, premiums, reserves, + incident dates, status transitions, medical/loss descriptions, and claimant PII. + +Fixture files may contain synthetic PII canaries because the detector needs raw +inputs. No result, report, failure artifact, or committed baseline may contain +those literals. + +## Public Surface Inventory + +TruthBench maintains a versioned manifest of decision-bearing and decision-sink +surfaces. A contract test compares the manifest with the public exports and known +CLI commands so a new surface cannot be added without an explicit coverage +classification. + +The release profile covers: + +- `fd.clean`, `Cleaner`, `clean_csv`, `CleanResult`, and `CleanReport`; +- fluent pipelines, plan/suggest/apply workflows, and decision hashes; +- field validation and remediation policy; +- validation suites, enterprise contracts, and context policies; +- bundled domain validators and domain repair integration; +- deterministic semantic default, review, and auto behavior; +- safe text cleaning and encoding linting; +- PII detection, anonymization, privacy policies, and privacy reports; +- trust scoring, quality reports, and trust gates; +- streaming with fixed batch partitions; +- pandas, Polars, and DuckDB required backend paths; +- AI Copilot with `provider=None` only; +- generated Copilot code parsing, compilation, and controlled execution; +- report exporters, JSON, Markdown, HTML/Peel rendering, CLI stdout/stderr, and + persisted artifacts as privacy and audit sinks. + +Explicit low-level utilities that perform a caller-requested transform but make no +semantic decision are classified as `explicit-transform`. They receive input/output +and safety contract tests, but they do not pretend to infer a disposition. + +## Surface Adapter Contract + +Every adapter declares: + +- stable name and version; +- whether it mutates, validates, suggests, serializes, or renders; +- supported fixture features and required dependencies; +- the surface-specific mapping from the global gold disposition to the expected + surface decision; +- output extraction, decision extraction, and audit extraction rules; +- deterministic fields and explicitly excluded telemetry fields; +- whether exact backend parity is required. + +For example, a mutating cleaner is expected to repair `repair`, preserve +`preserve`, and leave `flag`/`review` unchanged while surfacing them. A read-only +validator is expected to preserve values and emit `flag` or `review` evidence for +the corresponding defects. A PII detector is expected to flag labelled PII and +remain silent on non-PII preservation traps. + +Adapters must not catch broad exceptions. An unexpected exception is a benchmark +failure with its type, safe message, fixture, surface, and reproduction ID. + +## Normalized Decision Record + +TruthBench emits one record for every `(surface, backend, repeat, cell)` tuple: + +- record schema version; +- run, fixture, case, and cell IDs; +- domain, row ID, and column; +- expected and actual disposition; +- input type and privacy-safe digest; +- expected and actual output type; +- output value for non-sensitive cells or a redacted value plus digest for + sensitive cells; +- confidence, risk, status, model/rule ID, and rationale; +- evidence kinds without raw sensitive samples; +- mutation, detection, quarantine, and human-review booleans; +- audit-required and audit-complete booleans plus matching audit IDs; +- trust before, trust after, and delta at the applicable frame/surface level; +- requested backend, actual backend, fallback events, and backend differences; +- normalized decision hash and repeat-consistency status. + +Exact output equality includes dtype and semantic representation. The comparator +does not equate `"402.10"` with `402.1`, strip whitespace before preservation +checks, or treat case-folded identifiers as identical. Explicit fixture metadata +may authorize a representation equivalence only when that equivalence is itself +the gold repair. + +## Backend Consistency + +The release environment must install and successfully execute pandas, Polars, and +DuckDB. Missing required dependencies, silent materialization, unexpected fallback, +or a mismatched requested/actual backend fails the run. + +Parity uses the common deterministic native subset with +`fallback_policy="error"`. It compares normalized decisions, exact values where +the documented contract promises equality, row identity/order, action status, +confidence, rationale class, and audit coverage. A documented dtype or aggregation +difference is only accepted when represented by an explicit, tested equivalence +rule and disclosed in the report. + +Spark and FreshCore require JVM/native build infrastructure. They belong to the +extended scheduled profile, where absence is a profile failure rather than a skip. +Their adapter contracts and gate-tampering tests remain part of normal pytest. + +## Determinism + +Each deterministic surface runs at least twice in a fresh logical context. +TruthBench compares normalized data, decisions, confidences, rationales, findings, +audit records, ordering, decision hashes, and trust scores. + +Wall-clock duration, peak memory, generated timestamps, run IDs, lineage IDs, and +documented cryptographic randomness are excluded from the decision hash. Privacy +operations that intentionally use random salts must still make the same masking +decision, redact the same spans, disclose the randomness, and produce no raw PII. +Parity tests use an explicit fixed secret so output equality remains testable. + +The default cleaning path, default semantic backend, validators, domain decisions, +trust scores, and generated code must be fully deterministic. + +## Privacy Boundary + +The raw-PII gate scans every sink that could escape the input data boundary: + +- serialized reports and findings; +- action metadata, coerced-cell recovery records, examples, and audit logs; +- Copilot model context, narrative inputs, recommended code, and rendered report; +- plan JSON, validation JSON, domain logs, privacy reports, and quality reports; +- CLI stdout/stderr, Markdown, HTML, JSON, exception tables, and committed results; +- minimized reproductions and failure catalogues. + +The scanner checks exact canaries plus case, whitespace, punctuation, encoding, +Unicode-normalization, and digit-only variants. Sensitive values are represented by +`[REDACTED]` and a one-way run-scoped digest. Explicit in-memory APIs whose purpose +is to return the user's own data remain usable, but their default serialization and +rendering paths must not leak raw PII. Any explicit `include_pii=True`-style escape +hatch is outside the release profile and receives a separate opt-in warning test. + +## Runner Flow + +For each profile, seed, fixture, and surface, the runner: + +1. validates the complete oracle and fixture hash; +2. snapshots the input and protected columns; +3. computes pristine and adversarial trust scores; +4. invokes the surface through its public API; +5. extracts output, decisions, audit evidence, and backend disclosures; +6. verifies input immutability and protected-column byte identity; +7. emits normalized cell records and schema/row case records; +8. reruns deterministic surfaces and compares normalized hashes; +9. compares required backends; +10. scans every sink for PII canaries; +11. evaluates absolute gates; +12. minimizes every failure and writes privacy-safe reproduction artifacts; +13. writes versioned JSON and Markdown results atomically. + +Infrastructure errors fail closed. A partial run cannot report passed gates. + +## Mandatory Release Gates + +The release command fails if any of these conditions is true: + +1. **Valid-value corruption:** any `preserve` cell changes value, dtype, row + identity, or semantic representation, or receives an error-severity false alarm. +2. **Protected-column modification:** any protected cell or protected schema + property differs, even if another oracle labelled the value repairable. +3. **Raw PII leakage:** any canary or normalized variant appears in a scanned sink. +4. **Backend inconsistency:** any required backend produces a divergent decision, + output, ordering, confidence class, rationale class, or audit outcome without an + explicit permitted equivalence. +5. **Default non-determinism:** repeated default decisions or normalized outputs + differ. +6. **Broken generated code:** generated code fails AST parsing, compilation, or + controlled execution against its fixture, performs a network call, or exposes a + PII canary. +7. **Unexplained high confidence:** a decision at confidence `>= 0.90` lacks a + non-empty substantive rationale, rule/model provenance, and matching audit + evidence. Boilerplate or whitespace-only text does not count. +8. **Trust inversion:** an adversarial/corrupted frame scores higher than its + pristine source, or a destructive degenerate output is scored as trustworthy. + +Additional correctness gates require complete cell labels, zero unexpected +exceptions, zero unresolved required backends, 100% audit coverage for mutations, +100% routing of `review` cases, exact repair outputs, zero mutations of `flag` +cases, and internally consistent result counts. + +Absolute gates cannot be waived by a historical baseline. Baselines may detect +additional regressions but cannot make a mandatory failure pass. + +## Failure Reproduction and Minimization + +Every failure receives a stable ID derived from fixture version, surface, backend, +cell/case ID, gate, and normalized evidence. The reducer repeatedly tests the same +public surface while attempting, in order: + +1. removal of unrelated fixtures and policies; +2. removal of unrelated columns; +3. removal of unrelated rows while retaining required role/context evidence; +4. removal of unrelated labelled mutations; +5. simplification of values, schemas, and policy sentences; +6. reduction to one backend and one repeat when those dimensions are not causal. + +The reducer stops at a local one-minimal case or a configured evaluation budget. +It never changes the expected disposition or treats disappearance of the target +cell as success. The minimized artifact includes a sanitized frame, schema, policy, +config, exact command, expected/actual record, likely component, and result hash. + +For every validated implementation failure, the development workflow is: + +1. reproduce through TruthBench; +2. minimize; +3. identify the source-to-decision root cause; +4. add a focused failing pytest regression; +5. implement one root-cause fix; +6. run the focused test, affected suite, TruthBench, and full required tests; +7. update committed TruthBench results and failure catalogue; +8. record the fix, regression test, and remaining limitation. + +Expected outputs are changed only when the written product contract changes and a +human explicitly approves that contract change. They are never changed to conceal +a failure. + +## Result Artifacts + +Committed artifacts live under `benchmarks/truthbench/results/`: + +- `latest.json`: versioned complete machine-readable result; +- `latest.md`: concise metrics, gate status, failures, fixes, and limitations; +- `baseline.json`: fixture/result hashes and regression metrics, not gate waivers; +- `failures/.json`: minimized privacy-safe reproductions for unresolved + or newly fixed failures; +- `README.md`: schema, reproduction, comparison, and update policy. + +The JSON records repository commit, FreshData version, Python and dependency +versions, operating system, profile, seeds, fixture hashes, surface versions, +required backends, gate configuration, command, and normalized decision hashes. +Timestamps are metadata and never part of reproducibility hashes. + +Result verification rejects unknown schema versions, missing fixtures, incomplete +runs, mismatched fixture hashes, inconsistent aggregates, leaked PII, absent gate +results, and claims not backed by cell records. + +## CI and Release Integration + +Normal pytest receives fast contract tests for models, fixture completeness, +adapters, comparators, gates, minimization, serialization, privacy scanning, +generated code, and every fixed implementation failure. + +PR CI runs the deterministic release profile: + +```bash +python -m benchmarks.truthbench run \ + --profile release \ + --backends pandas,polars,duckdb \ + --require-backends \ + --repeats 2 \ + --check +``` + +The production release workflow runs the same command against the exact resolved +release commit before building distributions. A failure prevents publication. + +A scheduled extended profile adds Spark, FreshCore, more seeds, more generated +variants, alternative batch partitions, and larger fixtures. It has no +`continue-on-error`, no silent optional-dependency skip, and publishes its full +result artifacts. + +The Makefile exposes `truthbench`, `truthbench-release`, and +`truthbench-extended` targets. Documentation gives exact local reproduction +commands and distinguishes semantic correctness from performance evidence. + +## Testing Strategy + +TruthBench tests are layered: + +- model invariants and JSON schema round trips; +- complete cell-label and stable-ID validation; +- deterministic fixture and adversarial generator tests; +- exact comparator tests for values, dtypes, Unicode, missing values, dates, and + protected bytes; +- adapter contract tests using small synthetic surface outputs; +- public-surface inventory completeness tests; +- gate-tampering tests proving each mandatory gate fails independently; +- PII scanner mutation tests for every normalized canary variant; +- backend parity and fallback-honesty tests; +- repeated-run determinism tests; +- generated-code parse, compile, controlled-execution, network-denial, and privacy + tests; +- reducer tests proving the target failure remains after minimization; +- end-to-end smoke and full release-profile tests; +- one focused permanent regression for every reproduced FreshData defect. + +Tests must not use broad exception catches, unconditional skips, weakened existing +assertions, or changed gold outputs to make a run green. Optional infrastructure is +selected by profile; within a selected profile it is required. + +## Initial Audit Hypotheses + +Repository inspection identified high-priority hypotheses that TruthBench must +attempt to reproduce before they are called defects: + +- domain repair may occur after the core protected-column verification; +- post-domain report totals may describe the pre-domain frame; +- the 1,000-entry `coerced_cells` cap may also cap quarantine protection and allow + later coerced cells to be imputed; +- semantic review mode may auto-apply high-confidence proposals from non-default + backends; +- Validation Gauntlet semantic actions may not map to cells because semantic + actions aggregate distinct values rather than carrying a row; +- Validation Gauntlet domain findings may be counted without earning cell-level + detection evidence; +- column-level audit attribution may overstate cell-level completeness; +- serialized raw recovery values, text-cleaning originals, validation originals, + plan examples, or privacy groups may cross the intended audit privacy boundary; +- generated timestamps and random masking salts may be confused with decision + non-determinism unless normalization is explicit; +- plan signatures that sample only early rows may miss tail-only drift. + +Each hypothesis is subject to the reproduce/minimize/root-cause workflow. No source +change is justified solely by inspection. + +## Acceptance Criteria + +TruthBench is complete when: + +- all eight domain fixtures exist and every cell has one valid disposition; +- the public-surface manifest has no unclassified decision-bearing surface; +- every required comparison dimension appears in each applicable decision record; +- pandas, Polars, and DuckDB complete the release profile with no inconsistent + decisions or undisclosed fallback; +- all eight mandatory gates have independent negative tests and pass the real run; +- all discovered implementation failures have minimized reproductions, root-cause + fixes, focused pytest regressions, and updated benchmark results; +- generated code compiles and executes in the controlled offline harness; +- committed artifacts contain no planted PII; +- default behavior is deterministic under normalized repeat comparison; +- trust never increases after labelled corruption; +- the existing test suite, Validation Gauntlet, CleanBench, and benchmark tests are + not weakened; +- PR CI and the production release workflow execute TruthBench as a required gate; +- final documentation lists implemented files, failures, fixes, commands, results, + limitations, and evidence for every release gate. + From 0b2a8234cd2b1975afc04de54f17d88d487e0b55 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 10:51:39 +0530 Subject: [PATCH 02/40] docs: plan FreshData TruthBench implementation --- .../plans/2026-07-14-freshdata-truthbench.md | 687 ++++++++++++++++++ 1 file changed, 687 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-14-freshdata-truthbench.md diff --git a/docs/superpowers/plans/2026-07-14-freshdata-truthbench.md b/docs/superpowers/plans/2026-07-14-freshdata-truthbench.md new file mode 100644 index 0000000..5a77975 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-freshdata-truthbench.md @@ -0,0 +1,687 @@ +# FreshData TruthBench Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a deterministic, privacy-safe semantic red-team and regression system that gives every test cell a gold disposition, exercises every public FreshData decision surface, minimizes failures, and blocks releases on the eight approved safety gates. + +**Architecture:** Add an independent `benchmarks.truthbench` package with immutable oracle models, eight deterministic domain fixtures, exact typed comparison, public-surface adapters, normalized records, privacy scanning, absolute gates, repeat/backend comparison, generated-code isolation, deterministic minimization, and atomic result reporting. Keep all model-assisted activity outside FreshData's runtime; release runs call only deterministic public APIs and Copilot with `provider=None`. Source fixes are allowed only after a TruthBench reproduction and a focused failing pytest regression establish the defect. + +**Tech Stack:** Python 3.9+, pandas, Polars, DuckDB, FreshData public APIs, dataclasses/enums, hashlib/HMAC, `jsonschema`, `ast`, `subprocess`, pytest, GitHub Actions. + +## Global Constraints + +- Work only in `/Users/wilson/freshdata-worktrees/truthbench-jwd` on `feature/truthbench-jwd`; do not modify `/Users/wilson/freshdata`. +- Keep GPT/model/provider calls out of `fd.clean`, semantic routing, domain repair, privacy processing, trust scoring, CI, and TruthBench release execution. +- Do not weaken, skip, rebaseline, or rewrite existing tests or gold outputs to obtain a pass. +- Retain the approved baseline note: the two one-off `tests/test_benchmark.py` throughput failures are pre-existing environment timing noise; both exact reruns passed. Do not change their thresholds. +- A required backend that is unavailable or falls back is a failure, not a skip. +- Never write raw fixture PII into result JSON, Markdown, logs, exceptions, minimized artifacts, or committed baselines. +- Use one focused commit per task or validated source defect. Run the named focused test before every commit. +- For every implementation defect: reproduce, minimize, explain root cause, add a permanent regression, fix the root cause, rerun the affected suite, and update TruthBench artifacts. + +## Planned File Structure + +```text +benchmarks/truthbench/ + __init__.py # supported public harness API + __main__.py # module entry point + cli.py # release/extended/check command line + models.py # immutable oracle/result/gate models + exact.py # typed exact values and equality + schema.py # JSON schema and aggregate validation + privacy.py # canary variants, redaction, sink scanner + inventory.py # classified public-surface manifest + fixtures/ + __init__.py # registry and build_fixture + base.py # fixture builder and completeness validation + finance.py + healthcare.py + retail.py + crm.py + logistics.py + government.py + education.py + insurance.py + surfaces/ + __init__.py # adapter registry + base.py # adapter protocol and observation envelope + cleaning.py # clean/Cleaner/CSV/pipeline/plan/streaming + validation.py # field/suite/context/domain/contracts/text + privacy.py # PII/anonymization/privacy policy + reporting.py # reports/findings/export/render/CLI sinks + backends.py # pandas/Polars/DuckDB parity execution + copilot.py # provider=None and generated-code harness + normalize.py # observations -> per-cell decision records + gates.py # absolute release gates + determinism.py # stable decision hashes and repeat comparison + generated_code.py # offline AST/compile/subprocess execution + minimize.py # deterministic one-minimal failure reducer + runner.py # end-to-end orchestration + report.py # atomic JSON/Markdown/baseline output + results/ + README.md + baseline.json + latest.json + latest.md + failures/.gitkeep +tests/truthbench/ + conftest.py + test_models_exact.py + test_fixtures.py + test_privacy.py + test_inventory.py + test_surface_adapters.py + test_normalize.py + test_gates.py + test_backends_determinism.py + test_generated_code.py + test_minimize.py + test_runner_cli.py +tests/regressions/ + test_truthbench_domain_guard.py + test_truthbench_domain_report.py + test_truthbench_quarantine.py + test_truthbench_semantic_review.py + test_truthbench_findings_audit.py +docs/truthbench.md +.github/workflows/truthbench-extended.yml +``` + +Existing files changed only where the failing tests justify it: `src/freshdata/api.py`, `src/freshdata/report.py`, `src/freshdata/steps/dtypes.py`, `src/freshdata/engine/missing.py`, `src/freshdata/semantic/policy.py`, `src/freshdata/findings.py`, `Makefile`, `.github/workflows/ci.yml`, and `.github/workflows/release.yml`. + +--- + +### Task 1: Establish the immutable oracle and exact typed comparator + +**Files:** +- Create: `benchmarks/truthbench/__init__.py` +- Create: `benchmarks/truthbench/models.py` +- Create: `benchmarks/truthbench/exact.py` +- Create: `tests/truthbench/test_models_exact.py` + +**Interfaces:** Consumes Python/pandas scalar values and fixture metadata. Produces frozen `GoldCell`, `CaseExpectation`, `DecisionRecord`, `GateResult`, and `RunResult` objects plus canonical JSON-safe typed values. No FreshData runtime dependency. + +- [ ] Write failing tests covering the four-only disposition enum, stable cell IDs, sensitive-value redaction, JSON round trips, and exact distinctions among `"402.10"`, `402.1`, `np.float64(402.1)`, `None`, `pd.NA`, `NaN`, NFC/NFD Unicode, timezone-aware timestamps, leading-zero IDs, and categorical/string dtypes. + +```python +def test_exact_values_do_not_use_gauntlet_canonicalization(): + assert not exact_equal("402.10", 402.1) + assert not exact_equal(" AAPL", "AAPL") + assert not exact_equal("AAPL", "aapl") + assert exact_equal(pd.NA, pd.NA) + +def test_sensitive_record_never_serializes_raw_value(): + cell = GoldCell.create("v1", "crm", "r7", "notes", "flag", sensitive=True) + record = DecisionRecord.for_test(cell=cell, input_value="tb.person+7@example.invalid") + payload = record.to_dict() + assert "tb.person+7@example.invalid" not in json.dumps(payload) + assert payload["input"]["display"] == "[REDACTED]" +``` + +- [ ] Run `PYTHONPATH=src python -m pytest tests/truthbench/test_models_exact.py -q`; expect import/collection failure because the package does not exist. + +- [ ] Implement `Disposition(StrEnum)`, frozen dataclasses with explicit `schema_version=1`, `GoldCell.create()` stable ID generation, `TypedValue`, and a `canonical_json()` serializer that rejects non-finite JSON numbers and unknown types. + +```python +class Disposition(str, Enum): + PRESERVE = "preserve" + REPAIR = "repair" + FLAG = "flag" + REVIEW = "review" + +def exact_equal(left: Any, right: Any) -> bool: + return encode_typed(left) == encode_typed(right) + +def stable_digest(value: Any, *, key: bytes) -> str: + encoded = canonical_json(encode_typed(value)).encode("utf-8") + return hmac.new(key, encoded, hashlib.sha256).hexdigest() +``` + +- [ ] Ensure every `DecisionRecord` includes expected/actual disposition, input/output type, confidence, rationale, audit, trust, requested/actual backend, fallback events, and repeat hash fields; represent non-applicable dimensions explicitly as `None`, never by omission. + +- [ ] Rerun the focused test; expect all tests to pass. + +- [ ] Commit: `git add benchmarks/truthbench tests/truthbench/test_models_exact.py && git commit -m "feat: add TruthBench oracle models"` + +### Task 2: Add JSON schema and aggregate integrity validation + +**Files:** +- Create: `benchmarks/truthbench/schema.py` +- Create: `tests/truthbench/test_schema.py` + +**Interfaces:** Consumes serialized `RunResult`. Produces either a validated payload or a precise `TruthBenchSchemaError`. It must reject partial runs and inconsistent aggregates before gates are evaluated. + +- [ ] Write failing tests for unknown schema versions, absent fixtures/backends/gates, duplicate record IDs, non-finite confidence/trust values, fixture-hash mismatches, incorrect aggregate counts, and an `overall_passed=True` claim with a failed gate. + +- [ ] Run `PYTHONPATH=src python -m pytest tests/truthbench/test_schema.py -q`; expect failures for missing schema validation. + +- [ ] Implement a Draft 2020-12 schema with `additionalProperties: false` at result, record, gate, failure, and environment levels. Add semantic post-validation: + +```python +def validate_run(payload: Mapping[str, Any]) -> None: + jsonschema.Draft202012Validator(RESULT_SCHEMA).validate(payload) + ids = [record["record_id"] for record in payload["records"]] + if len(ids) != len(set(ids)): + raise TruthBenchSchemaError("duplicate decision record id") + if payload["summary"]["records"] != len(ids): + raise TruthBenchSchemaError("record aggregate does not match records") + passed = all(gate["passed"] for gate in payload["gates"]) + if payload["summary"]["overall_passed"] is not passed: + raise TruthBenchSchemaError("overall gate claim is inconsistent") +``` + +- [ ] Rerun the focused test; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/schema.py tests/truthbench/test_schema.py && git commit -m "feat: validate TruthBench result integrity"` + +### Task 3: Build fixture infrastructure with complete physical-cell labels + +**Files:** +- Create: `benchmarks/truthbench/fixtures/__init__.py` +- Create: `benchmarks/truthbench/fixtures/base.py` +- Create: `tests/truthbench/conftest.py` +- Create: `tests/truthbench/test_fixtures.py` + +**Interfaces:** Consumes a domain name and fixed seed. Produces a `TruthFixture` holding pristine/adversarial pandas frames, one `GoldCell` for every physical adversarial-frame cell, schema, policy, protected columns, PII canaries, row cases, schema cases, and a deterministic fixture hash. + +- [ ] Write failing fixture invariant tests. The label count must equal `rows * columns`; every `(row_id, column)` appears exactly once; injected cases overwrite the default `preserve` label; every repair has an exact typed output; every sensitive cell has a canary ID; row/schema expectations never stand in for cell labels. + +```python +@pytest.mark.parametrize("domain", DOMAINS) +def test_every_physical_cell_has_exactly_one_label(domain): + fixture = build_fixture(domain, seed=1729) + expected = {(str(row), str(col)) for row in fixture.frame.index for col in fixture.frame} + actual = {(cell.row_id, cell.column) for cell in fixture.cells} + assert actual == expected + assert len(fixture.cells) == len(expected) + fixture.validate() +``` + +- [ ] Run `PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q`; expect import failure. + +- [ ] Implement `FixtureBuilder` so it starts with a complete `preserve` label matrix and `inject()` atomically changes a value and replaces exactly one label. Reject missing/duplicate row IDs, unknown columns, non-synthetic PII domains, and contradictory repair outputs. + +```python +class FixtureBuilder: + def inject(self, row_id: str, column: str, value: Any, disposition: Disposition, + *, expected: Any = UNSET, family: str, sensitive: bool = False) -> None: + key = (row_id, column) + if key not in self._labels: + raise FixtureError(f"unknown cell {key}") + self.frame.at[row_id, column] = value + self._labels[key] = GoldCell.create( + self.version, self.domain, row_id, column, disposition, + expected_output=expected, family=family, sensitive=sensitive, + ) +``` + +- [ ] Make `fixture_hash` cover typed pristine/adversarial values, labels, schema, policy, row/schema cases, and protected columns, excluding object identity and build time. + +- [ ] Rerun the focused test with a temporary minimal fixture registered in `conftest.py`; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/fixtures tests/truthbench/conftest.py tests/truthbench/test_fixtures.py && git commit -m "feat: add complete TruthBench fixture oracle"` + +### Task 4: Add finance, healthcare, retail, and CRM gold datasets + +**Files:** +- Create: `benchmarks/truthbench/fixtures/finance.py` +- Create: `benchmarks/truthbench/fixtures/healthcare.py` +- Create: `benchmarks/truthbench/fixtures/retail.py` +- Create: `benchmarks/truthbench/fixtures/crm.py` +- Modify: `benchmarks/truthbench/fixtures/__init__.py` +- Modify: `tests/truthbench/test_fixtures.py` + +**Interfaces:** Each `build(seed: int) -> TruthFixture` is pure and deterministic. Cases use only reserved synthetic namespaces such as `.invalid`, `555-01xx`, and explicit `TB-*` identifiers. + +- [ ] Add failing domain-content tests requiring each disposition in every domain and the following exact adversarial families: + +| Domain | Required cases | +|---|---| +| finance | `apple` in price, `Apple` company, `AAPL` ticker; `0.00`; negative/extreme values; `₹1,23,456.70`; USD/EUR/INR conflict; `01/02/2025`; invisible PII in memo; tail-row account canary; protected ticker policy conflict | +| healthcare | valid rare ICD/LOINC; `98.6` in Celsius; `5 mg`/`5000 mcg`; partial/FHIR/impossible dates; decomposed Unicode name; MRN/phone/PHI in notes; protected DOB repair conflict | +| retail | leading-zero SKU/GTIN; free item and return quantity; mixed decimal/grouping/currency; HTML/entity/mojibake review; multilingual product; card/email in review; added/reordered/type-drifted columns | +| CRM | Unicode/combining names; `.invalid` email; spaced phone; ambiguous country/language/date; lifecycle contradiction; zero-width email and hidden SSN; `apple` lead source versus `Apple` employer; protected customer ID | + +- [ ] Run `PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q`; expect four missing builders/content failures. + +- [ ] Implement 16-row pristine frames per domain with stable string indexes and at least 12 injected cases per domain. Use fixed reference date `2026-01-15`, UTC timezone, and explicit locale metadata. Keep unusual valid values labelled `preserve`, deterministic representational fixes labelled `repair`, unsafe values labelled `flag`, and ambiguous/policy-conflicted values labelled `review`. + +- [ ] Add row cases for exact duplicates and schema cases for added/removed/renamed/reordered/type-drifted columns. Do not label a removed row/column as a cell outcome. + +- [ ] Assert byte-for-byte deterministic frame serialization and fixture hashes over two builds for seeds `1729` and `2718`. + +- [ ] Rerun the fixture tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py && git commit -m "feat: add first TruthBench domain corpus"` + +### Task 5: Add logistics, government, education, and insurance gold datasets + +**Files:** +- Create: `benchmarks/truthbench/fixtures/logistics.py` +- Create: `benchmarks/truthbench/fixtures/government.py` +- Create: `benchmarks/truthbench/fixtures/education.py` +- Create: `benchmarks/truthbench/fixtures/insurance.py` +- Modify: `benchmarks/truthbench/fixtures/__init__.py` +- Modify: `tests/truthbench/test_fixtures.py` + +**Interfaces:** Same builder contract as Task 4; registry order is the stable alphabetical order `crm, education, finance, government, healthcare, insurance, logistics, retail`. + +- [ ] Add failing content tests for: + +| Domain | Required cases | +|---|---| +| logistics | rare valid UN/LOCODE-like code; kg/lb and C/F; cross-timezone delivery window; 24:00-like transport time; address PII; late tracking canary; protected shipment ID conflict | +| government | leading-zero district/case IDs; Indian/international grouping; fiscal versus calendar year; multilingual agency names; restricted national ID in notes; mixed legacy encoding; contradictory retention/repair policy | +| education | student IDs; letter/percentage/GPA scales; school-year ambiguity; valid zero score; enrollment date ordering; guardian email/phone and FERPA notes; protected student ID and grade policy conflict | +| insurance | policy/claim IDs; premium/reserve currency conflict; negative reserve review; incident/report date ordering; state transition contradiction; claimant PII and medical loss text; protected policy number conflict | + +- [ ] Run the fixture suite and expect four missing-builder/content failures. + +- [ ] Implement the four deterministic builders using the same 16-row/fixed-reference conventions, complete labels, row cases, schema drift, mixed language/encoding, and PII tail cases. + +- [ ] Add a corpus-level assertion that all required trap categories occur across the eight fixtures and every domain contains all four dispositions. + +- [ ] Rerun `PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q`; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py && git commit -m "feat: complete eight-domain TruthBench corpus"` + +### Task 6: Implement privacy-safe values and exhaustive sink scanning + +**Files:** +- Create: `benchmarks/truthbench/privacy.py` +- Create: `tests/truthbench/test_privacy.py` + +**Interfaces:** Consumes fixture canaries and arbitrary nested sink values (`str`, bytes, mappings, sequences, dataclasses, pandas objects). Produces redacted values/digests and precise leak locations without repeating leaked text. + +- [ ] Write failing mutation tests for literal, case-folded, whitespace-stripped, punctuation-stripped, digit-only, URL-decoded, HTML-unescaped, UTF-8 bytes, NFKC/NFC/NFD, zero-width-removed, and JSON-escaped canary variants. + +```python +@pytest.mark.parametrize("mutate", CANARY_MUTATORS) +def test_scanner_finds_every_normalized_variant(mutate): + scanner = SinkScanner.from_canaries({"crm-email": "tb.person+7@example.invalid"}) + leaks = scanner.scan({"report": [mutate("tb.person+7@example.invalid")]}) + assert [(leak.canary_id, leak.path) for leak in leaks] == [("crm-email", "$.report[0]")] + assert "tb.person" not in repr(leaks) +``` + +- [ ] Run the focused tests; expect import failure. + +- [ ] Implement normalization as named transforms and scan both decoded text and byte hex/escape forms. Use run-scoped HMAC-SHA256 digests and `Leak(canary_id, variant, path)`; never store a matched substring. + +- [ ] Add scanners for exception text, `CleanReport.to_dict()`, action metadata, `coerced_cells`, findings, plan JSON, validation/domain/privacy/Copilot reports, generated code, stdout/stderr, Markdown, HTML, JSON, and failure artifacts. + +- [ ] Verify scanner self-test rejects a result that contains its own canary and passes a correctly redacted sink. + +- [ ] Rerun focused tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/privacy.py tests/truthbench/test_privacy.py && git commit -m "feat: add TruthBench PII leak scanner"` + +### Task 7: Classify every public surface and define the adapter protocol + +**Files:** +- Create: `benchmarks/truthbench/inventory.py` +- Create: `benchmarks/truthbench/surfaces/__init__.py` +- Create: `benchmarks/truthbench/surfaces/base.py` +- Create: `tests/truthbench/test_inventory.py` +- Create: `tests/truthbench/test_surface_adapters.py` + +**Interfaces:** Consumes `freshdata.__all__`, lazy export registries, domain registry, experimental Copilot export, and enterprise CLI parser. Produces one classification for every public name/command: `decision`, `sink`, `explicit-transform`, `data-model`, `configuration`, `registration`, or `out-of-scope-with-reason`. Decision/sink entries must reference an adapter. + +- [ ] Write failing inventory tests comparing the manifest to `fd.__dir__()`, `_ENTERPRISE_EXPORTS`, `_INTEGRATION_EXPORTS`, `_LEARNING_EXPORTS`, `_VALIDATION_EXPORTS`, bundled `domains.available()`, and CLI subcommands. Fail on a new unclassified public name or an adapterless decision/sink. + +- [ ] Run focused tests; expect missing manifest/protocol failures. + +- [ ] Implement frozen `SurfaceSpec` and abstract `SurfaceAdapter.observe(fixture, context) -> SurfaceObservation`. `SurfaceObservation` must carry output frame, raw decisions, audit sinks, trust, backend disclosure, generated code, captured stdout/stderr, and unexpected exception details. + +```python +@dataclass(frozen=True) +class SurfaceSpec: + name: str + version: int + classification: SurfaceClass + adapter: str | None + mutates: bool + deterministic: bool + backend_parity: bool + rationale: str +``` + +- [ ] Explicitly classify low-level `fill_missing`, `remove_outliers`, `resolve_duplicates`, `group_aggregate`, display setters, plugin registration, token vault primitives, and explicit detokenization as caller-directed surfaces; they receive safety/input-output checks but no inferred disposition credit. + +- [ ] Rerun focused tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/inventory.py benchmarks/truthbench/surfaces tests/truthbench/test_inventory.py tests/truthbench/test_surface_adapters.py && git commit -m "feat: inventory FreshData decision surfaces"` + +### Task 8: Implement cleaning, validation, domain, text, and streaming adapters + +**Files:** +- Create: `benchmarks/truthbench/surfaces/cleaning.py` +- Create: `benchmarks/truthbench/surfaces/validation.py` +- Modify: `benchmarks/truthbench/surfaces/__init__.py` +- Modify: `tests/truthbench/test_surface_adapters.py` + +**Interfaces:** Calls public FreshData APIs only. Produces `SurfaceObservation` without grading it. Adapter-specific expected mappings are explicit: mutators repair only `repair`; read-only validators never mutate; `flag` and `review` remain unchanged by default. + +- [ ] Write failing contract tests for `fd.clean`, `Cleaner.clean`, `clean_csv`, `CleanResult`, fluent `pipeline`, `suggest_plan`/`apply_plan`, `validate_fields`/`apply_field_policy`, `fd.validate`/`ValidationSuite`, context compile/validate, bundled domain validators, deterministic semantic assist/review/auto, text clean/lint, and fixed-partition `StreamingCleaner`/`clean_timeseries`. + +- [ ] Run focused tests; expect missing adapters. + +- [ ] Implement adapters with narrow exception capture at the top runner boundary only. Preserve the exact exception type and a scanner-sanitized message. Use public methods for reports and decisions; do not import internal cleaning functions. + +- [ ] Map aggregate actions to cells only when metadata supplies a row, a documented value mapping identifies exact matching cells, or the returned validation/domain rule supplies violation rows. Never award a column-wide action to every labelled cell. + +- [ ] Capture input snapshot, output, `report.actions`, findings, `coerced_cells`, domain reports/repairs, trust fields, plan hashes, recommendations, and rendered representations as separate scan sinks. + +- [ ] For streaming, run identical data through fixed partitions `(8, 8)` and `(5, 5, 6)` and retain batch/rolling/cumulative decisions for later parity comparison. + +- [ ] Rerun focused adapter tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/surfaces tests/truthbench/test_surface_adapters.py && git commit -m "feat: observe core FreshData surfaces"` + +### Task 9: Implement privacy, trust, reporting, export, CLI, and Copilot adapters + +**Files:** +- Create: `benchmarks/truthbench/surfaces/privacy.py` +- Create: `benchmarks/truthbench/surfaces/reporting.py` +- Create: `benchmarks/truthbench/surfaces/copilot.py` +- Modify: `benchmarks/truthbench/surfaces/__init__.py` +- Modify: `tests/truthbench/test_surface_adapters.py` + +**Interfaces:** Calls `detect_pii`, anonymization/privacy policy, trust/quality/compliance/insight, findings/exporters/renderers/CLI, and `experimental.ai_copilot.analyze_dataset(provider=None)`. Produces observations plus every externally visible sink. + +- [ ] Add failing adapter tests for PII detection, anonymization with a fixed test key, privacy policy, k-anonymity, `compute_trust_score`, quality/debt/insight/compliance/stakeholder reports, `to_dict`/`to_frame`/`to_findings`, JSON/Markdown/HTML/Peel/plain rendering, quality-ops/dbt/GX/exception exporters, CLI stdout/stderr, and Copilot provider-free analysis. + +- [ ] Run focused tests; expect missing adapters. + +- [ ] Implement adapters. Copilot must receive `provider=None`; monkeypatch a sentinel provider/network function that fails if called. Record prompt/model context, recommended code, audit, narrative, and all render forms as privacy sinks. + +- [ ] Use a fixed per-test masking secret for deterministic parity, while separately asserting default random masking discloses randomness and never leaks raw PII. + +- [ ] Capture trust on pristine, adversarial, cleaned, and deliberately destructive frames. A destructive control is a same-shape constant/null frame, not an empty frame that bypasses metrics. + +- [ ] Rerun focused tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/surfaces tests/truthbench/test_surface_adapters.py && git commit -m "feat: observe privacy reporting and Copilot surfaces"` + +### Task 10: Add required-backend execution and honest parity checks + +**Files:** +- Create: `benchmarks/truthbench/surfaces/backends.py` +- Create: `tests/truthbench/test_backends_determinism.py` +- Modify: `benchmarks/truthbench/surfaces/__init__.py` + +**Interfaces:** Consumes a fixture and common-native-subset `CleanConfig`. Produces pandas-normalized observations for requested backends `pandas`, `polars`, and `duckdb` with `fallback_policy="error"`, requested/actual backend, fallback events, row identity/order, and report differences. + +- [ ] Write failing tests proving a missing required backend, unexpected fallback, requested/actual mismatch, row reorder, dtype/value divergence, action divergence, or undisclosed backend difference fails parity. Tamper adapters in tests; never rely on an actually missing dependency. + +- [ ] Run focused tests; expect missing backend adapter. + +- [ ] Implement backend preflight with `importlib.metadata.version`, public `fd.clean(..., engine=backend, fallback_policy="error", return_report=True)`, native-to-pandas conversion, and explicit equivalence rules limited to approved representation differences. + +- [ ] Ensure pandas is the reference but not automatically correct: all three outputs are independently scored against gold before parity comparison. + +- [ ] Add extended-profile contracts for Spark/FreshCore. Normal pytest tests adapter/gate behavior using fakes; the extended workflow requires real infrastructure. + +- [ ] Rerun focused tests with installed pandas/Polars/DuckDB; expect pass and no fallback events. + +- [ ] Commit: `git add benchmarks/truthbench/surfaces/backends.py tests/truthbench/test_backends_determinism.py benchmarks/truthbench/surfaces/__init__.py && git commit -m "feat: enforce TruthBench backend parity"` + +### Task 11: Normalize decisions and implement all absolute gates + +**Files:** +- Create: `benchmarks/truthbench/normalize.py` +- Create: `benchmarks/truthbench/gates.py` +- Create: `tests/truthbench/test_normalize.py` +- Create: `tests/truthbench/test_gates.py` + +**Interfaces:** Consumes fixture gold plus a `SurfaceObservation`. Produces one normalized `DecisionRecord` per `(surface, backend, repeat, cell)` and `CaseRecord` per row/schema case, then evaluates gates independently of any baseline. + +- [ ] Write failing normalization tests for exact output/dtype, detected versus mutated, quarantine/review routing, cell-level audit IDs, confidence/rationale/provenance extraction, trust delta, backend disclosure, sensitive values, and non-applicable fields. + +- [ ] Write one independent tamper test for each mandatory gate: + +```python +@pytest.mark.parametrize("mutator, gate", [ + (corrupt_preserve, "valid_value_corruption"), + (modify_protected, "protected_column_modification"), + (leak_canary, "raw_pii_leakage"), + (diverge_backend, "backend_inconsistency"), + (change_repeat, "default_nondeterminism"), + (break_generated_code, "broken_generated_code"), + (remove_high_confidence_explanation, "unexplained_high_confidence"), + (invert_trust, "trust_inversion"), +]) +def test_each_mandatory_gate_fails_independently(passing_run, mutator, gate): + result = evaluate_gates(mutator(passing_run)) + assert failed_gate_names(result) == {gate} +``` + +- [ ] Run focused tests; expect missing normalizer/gates. + +- [ ] Implement surface-aware disposition mapping. Mutators must exactly repair `repair`, preserve `preserve`, and not mutate `flag`/`review`. Validators receive detection credit without mutation. PII adapters are graded only on PII-labelled cells plus false positives. Explicit transforms are graded on requested behavior and protected/privacy invariants. + +- [ ] Implement the eight named gates plus completeness, unexpected exception, required-backend, mutation-audit, review-routing, exact-repair, flag-mutation, input-mutation, and aggregate-consistency gates. High confidence means `>= 0.90`; substantive explanations require non-boilerplate rationale, non-empty rule/model provenance, and a matching audit record. + +- [ ] Make gate evaluation fail closed when records are absent, a surface is unexecuted, schema validation failed, or a run is partial. Baseline comparison may add failures but cannot clear one. + +- [ ] Rerun focused tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/normalize.py benchmarks/truthbench/gates.py tests/truthbench/test_normalize.py tests/truthbench/test_gates.py && git commit -m "feat: enforce TruthBench release gates"` + +### Task 12: Add decision determinism and controlled generated-code execution + +**Files:** +- Create: `benchmarks/truthbench/determinism.py` +- Create: `benchmarks/truthbench/generated_code.py` +- Modify: `tests/truthbench/test_backends_determinism.py` +- Create: `tests/truthbench/test_generated_code.py` + +**Interfaces:** Consumes normalized observations and Copilot code. Produces stable hashes/diffs and a subprocess result that proves AST parse, compilation, offline execution, expected output, and PII safety. + +- [ ] Write failing tests showing hashes ignore duration, peak memory, timestamp, run/lineage IDs, and documented salt bytes but detect decision/order/output/confidence/rationale/audit/trust changes. + +- [ ] Write failing generated-code tests for syntax error, compile error, missing input/output contract, filesystem escape, imports outside the allowlist, `socket`/HTTP access, subprocess creation, raw PII literal, runtime failure, timeout, and wrong cleaned output. + +- [ ] Run the focused tests; expect missing modules. + +- [ ] Implement recursive normalization with an explicit excluded-field set and sorted canonical JSON. Reject callers attempting to exclude a decision-bearing field. + +- [ ] Parse code with `ast.parse`, reject unsafe nodes/imports/calls, compile in-process, then execute in `python -I` with a 10-second timeout, a temporary working directory, environment allowlist, network-denial bootstrap, and serialized synthetic fixture input. Compare the produced frame/report to the adapter contract and scan code/stdout/stderr/artifacts for canaries. + +- [ ] Rerun focused tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench/determinism.py benchmarks/truthbench/generated_code.py tests/truthbench/test_backends_determinism.py tests/truthbench/test_generated_code.py && git commit -m "feat: verify deterministic decisions and generated code"` + +### Task 13: Implement runner, deterministic minimizer, CLI, and atomic reports + +**Files:** +- Create: `benchmarks/truthbench/runner.py` +- Create: `benchmarks/truthbench/minimize.py` +- Create: `benchmarks/truthbench/report.py` +- Create: `benchmarks/truthbench/cli.py` +- Create: `benchmarks/truthbench/__main__.py` +- Create: `tests/truthbench/test_minimize.py` +- Create: `tests/truthbench/test_runner_cli.py` + +**Interfaces:** CLI accepts `run --profile release|extended --backends ... --require-backends --repeats N --check`. Runner returns a complete `RunResult`; minimizer consumes one reproducible `GateFailure` and predicate; reporter writes validated, PII-scanned JSON/Markdown atomically. + +- [ ] Write failing reducer tests proving removal order is fixtures/policies, columns, rows, mutations, value/schema/policy simplification, then backend/repeat. The target cell and expected disposition must survive and the same failure ID must still reproduce. + +- [ ] Write failing CLI tests for exact option parsing, unknown backend/profile, missing required backend, partial-run failure, nonzero `--check`, successful atomic replacement, and refusal to write a leaking artifact. + +- [ ] Run focused tests; expect missing runner/CLI/minimizer. + +- [ ] Implement the approved 13-stage runner flow. Use a fresh adapter context per repeat, deterministic fixture/surface ordering, no broad exception suppression, and an infrastructure-failure record when observation cannot complete. + +- [ ] Implement one-minimal reduction with an evaluation budget of 250 calls and a cache keyed by typed fixture/config/surface/backend hash. Generate failure IDs from fixture version, surface, backend, cell/case, gate, and normalized evidence. + +- [ ] Implement `write_atomic()` with same-directory temporary files, `fsync`, schema validation, sink scan, then `os.replace`. Render Markdown only from already-redacted JSON. + +- [ ] Add CLI defaults exactly matching the release command: + +```bash +python -m benchmarks.truthbench run --profile release \ + --backends pandas,polars,duckdb --require-backends --repeats 2 --check +``` + +- [ ] Rerun focused tests; expect pass. + +- [ ] Commit: `git add benchmarks/truthbench tests/truthbench/test_minimize.py tests/truthbench/test_runner_cli.py && git commit -m "feat: run and minimize TruthBench failures"` + +### Task 14: Reproduce and resolve the initial audit hypotheses + +**Files:** +- Create/modify only the regression and source files proven necessary by each reproduction. +- Expected focused files: `tests/regressions/test_truthbench_domain_guard.py`, `tests/regressions/test_truthbench_domain_report.py`, `tests/regressions/test_truthbench_quarantine.py`, `tests/regressions/test_truthbench_semantic_review.py`, `tests/regressions/test_truthbench_findings_audit.py`, `src/freshdata/api.py`, `src/freshdata/report.py`, `src/freshdata/steps/dtypes.py`, `src/freshdata/engine/missing.py`, `src/freshdata/semantic/policy.py`, `src/freshdata/findings.py`. + +**Interfaces:** Consumes real TruthBench failures. Produces minimized privacy-safe reproductions, root-cause notes, permanent regressions, minimal source fixes, and green affected suites. A hypothesis that does not reproduce gets a passing adversarial coverage test and no source change. + +- [ ] Run the release profile without `--check`, capture all failures, and run the minimizer for each: + +```bash +PYTHONPATH=src python -m benchmarks.truthbench run --profile release \ + --backends pandas,polars,duckdb --require-backends --repeats 2 +``` + +- [ ] For domain protection, add a focused test where a compiled policy marks a repairable domain column immutable. Expect byte-identical output or `ProtectedColumnError`; reproduce before changing `api.py`. If reproduced, snapshot the resolved protected columns before `run_domain`, verify the repaired frame afterward, and record the guard action. + +- [ ] For post-domain report state, assert `rows_after`, `cols_after`, `memory_after`, and `missing_after` describe the returned domain-repaired frame. If reproduced, centralize final metric refresh and call it after `_fold_domain_outcome`. + +- [ ] For late coerced cells, build 1,020 parse casualties where a row after index 1,000 would otherwise be imputed. If reproduced, add a non-serialized `_coerced_rows: dict[str, set[Any]]` to `CleanReport`, populate every lost index in `_record_coerced`, keep `coerced_cells` recovery values capped, and make `_quarantined_rows` use the complete internal index set. + +- [ ] For semantic review, inject a high-confidence low-risk proposal from `embedding`, `profile`, `memory`, and `plugin:*` backends. If any applies in review mode, restrict review-mode auto-application to deterministic built-in provenance; keep non-default proposals suggested with human review. Preserve `auto` behavior subject to its existing safety gates. + +- [ ] For finding audit projection, assert a medium/high-risk action retains confidence, rationale, model/rule ID, status, reversible, human-review, and safe metadata in `QualityFinding.extra`. If lost, pass complete action dictionaries from `CleanReport.to_findings()` and explicitly copy those safe audit fields in `findings_from_dict()`. + +- [ ] For every reproduced defect, follow this exact loop separately: run the failing focused test; save minimized redacted failure JSON; implement one root-cause fix; run the focused test; run the affected module suite; rerun its TruthBench case; commit with a concise `fix:` message naming the observed root cause. + +- [ ] Exercise the remaining hypotheses—Gauntlet row mapping, domain cell evidence, audit over-attribution, raw recovery leakage, random salt normalization, and tail-only plan drift—through TruthBench. Fix FreshData only if the new exact benchmark reproduces a public-contract violation; otherwise retain the adversarial test as coverage. + +- [ ] After each fix, scan the patch for broad catches/skips/changed gold: + +```bash +git diff origin/main -- tests src benchmarks/truthbench | rg "pytest\.skip|xfail|except Exception|COERCED_CELLS_CAP|expected|Disposition" +``` + +- [ ] Commit each validated fix independently; do not combine unrelated root causes. + +### Task 15: Commit benchmark artifacts and document every result + +**Files:** +- Create: `benchmarks/truthbench/results/README.md` +- Create: `benchmarks/truthbench/results/baseline.json` +- Create: `benchmarks/truthbench/results/latest.json` +- Create: `benchmarks/truthbench/results/latest.md` +- Create/update: `benchmarks/truthbench/results/failures/*.json` +- Create: `docs/truthbench.md` +- Modify: `README.md` + +**Interfaces:** Consumes the clean release run. Produces versioned, schema-valid, PII-free evidence and user documentation. Baseline detects regressions but cannot waive an absolute gate. + +- [ ] Add failing tests that committed artifacts validate, match current fixture hashes/surface manifest, contain all eight domain names and all gate results, contain no canary variants, and do not claim success for unresolved failures. + +- [ ] Run those tests; expect missing artifact failures. + +- [ ] Run the full release profile with `--check`; write `latest.json`/`latest.md`, then intentionally copy the verified result to `baseline.json` only after every absolute gate passes. + +- [ ] Document architecture, disposition meanings, surface mapping, exact comparator, privacy model, failure reproduction, minimization, baseline update policy, local commands, and the explicit prohibition on LLMs in the runtime path. + +- [ ] In `latest.md`, list implemented files, each discovered failure and root cause, each fix/regression, exact commands, per-domain/per-surface/backend results, every gate's evidence counts, remaining limitations, and the two pre-existing timing flakes with their passing exact reruns. + +- [ ] Rerun artifact tests and `rg` every planted canary across committed result/docs paths; expect zero matches outside fixture source and scanner test data. + +- [ ] Commit: `git add benchmarks/truthbench/results docs/truthbench.md README.md tests/truthbench && git commit -m "docs: publish TruthBench release evidence"` + +### Task 16: Make TruthBench a mandatory PR and production release gate + +**Files:** +- Modify: `Makefile` +- Modify: `.github/workflows/ci.yml` +- Modify: `.github/workflows/release.yml` +- Create: `.github/workflows/truthbench-extended.yml` +- Modify: `pyproject.toml` only if a dedicated benchmark extra is required after verification. + +**Interfaces:** CI/release consumes the repository and required dependencies. Produces a hard pass/fail before packaging/publication plus scheduled extended artifacts. + +- [ ] Add failing workflow/Makefile contract tests asserting exact required command, no `continue-on-error`, no skip-on-missing-backend branch, and TruthBench execution before `python -m build`/publication. + +- [ ] Add targets: + +```make +.PHONY: truthbench truthbench-release truthbench-extended +truthbench: + PYTHONPATH=src python -m benchmarks.truthbench run --profile release --backends pandas,polars,duckdb --require-backends --repeats 2 +truthbench-release: + PYTHONPATH=src python -m benchmarks.truthbench run --profile release --backends pandas,polars,duckdb --require-backends --repeats 2 --check +truthbench-extended: + PYTHONPATH=src python -m benchmarks.truthbench run --profile extended --backends pandas,polars,duckdb,spark,freshcore --require-backends --repeats 2 --check +``` + +- [ ] Install dev/out-of-core dependencies in PR/release jobs and add `make truthbench-release` after fast pytest and before build. The scheduled extended job provisions JVM/native infrastructure and uploads `latest.json`, `latest.md`, and minimized failures on success or failure. + +- [ ] Run workflow syntax/contract tests plus `make -n truthbench-release`; expect the exact command. + +- [ ] Commit: `git add Makefile .github/workflows tests pyproject.toml && git commit -m "ci: require TruthBench release gates"` + +### Task 17: Final verification and self-review + +**Files:** +- Review all changes from `origin/main...HEAD`. +- Update `benchmarks/truthbench/results/latest.json` and `latest.md` only if verification evidence changed. + +**Interfaces:** Consumes the finished branch. Produces reproducible evidence that the specification and every release gate are satisfied without test weakening. + +- [ ] Run focused TruthBench tests: + +```bash +PYTHONPATH=src python -m pytest tests/truthbench tests/regressions/test_truthbench_*.py -q +``` + +Expected: all pass, no skips. + +- [ ] Run the mandatory release benchmark: + +```bash +PYTHONPATH=src python -m benchmarks.truthbench run --profile release \ + --backends pandas,polars,duckdb --require-backends --repeats 2 --check +``` + +Expected: exit 0; all eight mandatory and additional completeness gates pass; no backend fallback; zero raw-PII findings. + +- [ ] Run existing correctness systems unchanged: + +```bash +PYTHONPATH=src python -m benchmarks.gauntlet run --check +PYTHONPATH=src python benchmarks/public_benchmark.py +PYTHONPATH=src python -m pytest -m "not online and not large" +``` + +Expected: Gauntlet gates pass, public/CleanBench benchmark passes, pytest passes apart from no accepted failures. If the two recorded timing tests flake, rerun their exact node IDs and report both outputs; do not alter thresholds. + +- [ ] Run static/package checks: + +```bash +ruff check . +mypy src/freshdata +python -m build +python -m twine check dist/* +``` + +Expected: all exit 0. + +- [ ] Perform specification traceability review: map every approved design section and acceptance criterion to an implemented file plus a passing test/result field. + +- [ ] Scan for incomplete implementation and policy violations: + +```bash +rg -n "TODO|FIXME|NotImplemented|pass$|pytest\.skip|xfail|continue-on-error" benchmarks/truthbench tests/truthbench tests/regressions .github/workflows +rg -n "openai|anthropic|provider=|requests\.|httpx\.|urllib" benchmarks/truthbench src/freshdata +``` + +Expected: no placeholders, hidden skips, release `continue-on-error`, or network/model calls in the benchmark/default path; the Copilot adapter contains only the explicit `provider=None` assertion and network-denial test harness. + +- [ ] Review `git diff --check`, `git status --short`, and the full commit list. Confirm the original checkout remains unchanged. + +- [ ] If verification changed result evidence, regenerate it, rerun schema/PII checks, and commit `test: finalize TruthBench release evidence`. + +- [ ] Prepare the final handoff with implemented files, discovered failures, fixes, commands and outputs, benchmark totals, limitations, and per-gate evidence. From 0c25acad85921d27778dd2d517a5447a27739b84 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 10:52:10 +0530 Subject: [PATCH 03/40] docs: normalize TruthBench spec formatting --- .../specs/2026-07-14-freshdata-truthbench-design.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md b/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md index 3cce5ba..f2e2bee 100644 --- a/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md +++ b/docs/superpowers/specs/2026-07-14-freshdata-truthbench-design.md @@ -1,8 +1,8 @@ # FreshData TruthBench Design -**Date:** 2026-07-14 -**Target:** `origin/main` at `a1da862` -**Implementation branch:** `feature/truthbench-jwd` +**Date:** 2026-07-14 +**Target:** `origin/main` at `a1da862` +**Implementation branch:** `feature/truthbench-jwd` **Status:** Approved for implementation planning ## Purpose @@ -496,4 +496,3 @@ TruthBench is complete when: - PR CI and the production release workflow execute TruthBench as a required gate; - final documentation lists implemented files, failures, fixes, commands, results, limitations, and evidence for every release gate. - From ce2ef9d1d076d78c3630de00078a4e5359d258b6 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 11:07:18 +0530 Subject: [PATCH 04/40] feat: add TruthBench oracle models --- benchmarks/truthbench/__init__.py | 27 ++ benchmarks/truthbench/exact.py | 239 ++++++++++++++++++ benchmarks/truthbench/models.py | 340 ++++++++++++++++++++++++++ tests/truthbench/test_models_exact.py | 194 +++++++++++++++ 4 files changed, 800 insertions(+) create mode 100644 benchmarks/truthbench/__init__.py create mode 100644 benchmarks/truthbench/exact.py create mode 100644 benchmarks/truthbench/models.py create mode 100644 tests/truthbench/test_models_exact.py diff --git a/benchmarks/truthbench/__init__.py b/benchmarks/truthbench/__init__.py new file mode 100644 index 0000000..b3e08a1 --- /dev/null +++ b/benchmarks/truthbench/__init__.py @@ -0,0 +1,27 @@ +"""Deterministic oracle and result primitives for FreshData TruthBench.""" + +from .exact import TypedValue, canonical_json, encode_typed, exact_equal, stable_digest +from .models import ( + UNSET, + CaseExpectation, + DecisionRecord, + Disposition, + GateResult, + GoldCell, + RunResult, +) + +__all__ = [ + "UNSET", + "CaseExpectation", + "DecisionRecord", + "Disposition", + "GateResult", + "GoldCell", + "RunResult", + "TypedValue", + "canonical_json", + "encode_typed", + "exact_equal", + "stable_digest", +] diff --git a/benchmarks/truthbench/exact.py b/benchmarks/truthbench/exact.py new file mode 100644 index 0000000..966a173 --- /dev/null +++ b/benchmarks/truthbench/exact.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from enum import Enum +from typing import Any + +import numpy as np +import pandas as pd + +JsonValue = Any + + +@dataclass(frozen=True) +class TypedValue: + """A scalar encoded without erasing its Python/pandas representation.""" + + kind: str + value: JsonValue + dtype: str | None + display: str + digest: str | None = None + redacted: bool = False + schema_version: int = field(default=1, init=False) + + @property + def type_label(self) -> str: + if self.dtype is None: + return self.kind + return f"{self.kind}[{self.dtype}]" + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "type": self.kind, + "dtype": self.dtype, + "value": self.value, + "display": self.display, + "digest": self.digest, + "redacted": self.redacted, + } + + @classmethod + def from_dict(cls, payload: Mapping[str, JsonValue]) -> TypedValue: + if payload.get("schema_version") != 1: + raise ValueError("unsupported typed value schema version") + return cls( + kind=str(payload["type"]), + dtype=None if payload["dtype"] is None else str(payload["dtype"]), + value=payload["value"], + display=str(payload["display"]), + digest=None if payload["digest"] is None else str(payload["digest"]), + redacted=bool(payload["redacted"]), + ) + + +def _dtype_name(dtype: Any, *, sensitive: bool) -> str | None: + if dtype is None: + return None + normalized = pd.api.types.pandas_dtype(dtype) + if isinstance(normalized, pd.CategoricalDtype): + categories = normalized.categories + if sensitive: + encoded_categories: JsonValue = {"count": len(categories)} + else: + encoded_categories = [encode_typed(value).to_dict() for value in categories] + category_json = canonical_json(encoded_categories) + ordered = "true" if normalized.ordered else "false" + return f"category[ordered={ordered};categories={category_json}]" + if isinstance(normalized, pd.StringDtype): + return f"string[{normalized.storage}]" + return str(normalized) + + +def _encode_scalar(value: Any) -> tuple[str, JsonValue, str]: + if value is pd.NA: + return "pandas.NA", None, "" + if value is pd.NaT: + return "pandas.NaT", None, "NaT" + if value is None: + return "python.none", None, "None" + + if isinstance(value, np.bool_): + scalar = bool(value) + return "numpy.bool_", scalar, str(scalar) + if isinstance(value, bool): + return "python.bool", value, str(value) + + if isinstance(value, np.integer): + return f"numpy.{value.dtype.name}", str(value), str(value) + if isinstance(value, int): + return "python.int", str(value), str(value) + + if isinstance(value, np.floating): + kind = f"numpy.{value.dtype.name}" + scalar = float(value) + if math.isnan(scalar): + return f"{kind}.nan", None, "NaN" + if math.isinf(scalar): + sign = "+" if scalar > 0 else "-" + return f"{kind}.infinity", sign, f"{sign}Infinity" + representation = repr(value.item()) + return kind, representation, representation + if isinstance(value, float): + if math.isnan(value): + return "python.float.nan", None, "NaN" + if math.isinf(value): + sign = "+" if value > 0 else "-" + return "python.float.infinity", sign, f"{sign}Infinity" + representation = repr(value) + return "python.float", representation, representation + + if isinstance(value, np.str_): + scalar = str(value) + return "numpy.str_", scalar, scalar + if isinstance(value, str): + return "python.str", value, value + + if isinstance(value, np.bytes_): + scalar = bytes(value).hex() + return "numpy.bytes_", scalar, repr(bytes(value)) + if isinstance(value, bytes): + return "python.bytes", value.hex(), repr(value) + + if isinstance(value, pd.Timestamp): + rendered = value.isoformat() + return "pandas.Timestamp", rendered, rendered + if isinstance(value, datetime): + rendered = value.isoformat() + return "python.datetime", rendered, rendered + if isinstance(value, date): + rendered = value.isoformat() + return "python.date", rendered, rendered + if isinstance(value, time): + rendered = value.isoformat() + return "python.time", rendered, rendered + + if isinstance(value, pd.Timedelta): + rendered = value.isoformat() + return "pandas.Timedelta", rendered, rendered + if isinstance(value, np.datetime64): + return f"numpy.{value.dtype}", str(value), str(value) + if isinstance(value, np.timedelta64): + return f"numpy.{value.dtype}", str(value), str(value) + if isinstance(value, timedelta): + rendered = repr(value) + return "python.timedelta", rendered, rendered + if isinstance(value, Decimal): + rendered = str(value) + return "python.Decimal", rendered, rendered + + raise TypeError(f"unsupported scalar type: {type(value).__name__}") + + +def encode_typed( + value: Any, + *, + dtype: Any = None, + sensitive: bool = False, + digest_key: bytes | None = None, +) -> TypedValue: + kind, encoded, display = _encode_scalar(value) + dtype_name = _dtype_name(dtype, sensitive=sensitive) + if not sensitive: + return TypedValue(kind=kind, value=encoded, dtype=dtype_name, display=display) + if digest_key is None: + raise ValueError("digest_key is required for sensitive values") + raw = TypedValue(kind=kind, value=encoded, dtype=dtype_name, display=display) + digest = hmac.new( + digest_key, + canonical_json(raw).encode("utf-8"), + hashlib.sha256, + ).hexdigest() + return TypedValue( + kind=kind, + value=None, + dtype=dtype_name, + display="[REDACTED]", + digest=digest, + redacted=True, + ) + + +def exact_equal( + left: Any, + right: Any, + *, + left_dtype: Any = None, + right_dtype: Any = None, +) -> bool: + return encode_typed(left, dtype=left_dtype) == encode_typed(right, dtype=right_dtype) + + +def _json_safe(value: Any, path: str = "$") -> JsonValue: + if isinstance(value, TypedValue): + return _json_safe(value.to_dict(), path) + if isinstance(value, Enum): + return _json_safe(value.value, path) + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError(f"{path} contains a non-finite number") + return value + if isinstance(value, Mapping): + result: dict[str, JsonValue] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise TypeError(f"{path} contains an unsupported non-string key") + result[key] = _json_safe(item, f"{path}.{key}") + return result + if isinstance(value, (list, tuple)): + return [_json_safe(item, f"{path}[{index}]") for index, item in enumerate(value)] + raise TypeError(f"{path} contains unsupported type: {type(value).__name__}") + + +def canonical_json(value: Any) -> str: + return json.dumps( + _json_safe(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def stable_digest(value: Any, *, key: bytes) -> str: + if not isinstance(key, bytes): + raise TypeError("key must be bytes") + encoded = canonical_json(encode_typed(value)).encode("utf-8") + return hmac.new(key, encoded, hashlib.sha256).hexdigest() diff --git a/benchmarks/truthbench/models.py b/benchmarks/truthbench/models.py new file mode 100644 index 0000000..0ff5132 --- /dev/null +++ b/benchmarks/truthbench/models.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from .exact import JsonValue, TypedValue, encode_typed + + +class Disposition(str, Enum): + PRESERVE = "preserve" + REPAIR = "repair" + FLAG = "flag" + REVIEW = "review" + + +class _Unset: + pass + + +UNSET = _Unset() +_TEST_DIGEST_KEY = b"truthbench-decision-record-for-test-v1" + + +def _disposition(value: Disposition | str) -> Disposition: + try: + return Disposition(value) + except ValueError as exc: + raise ValueError(f"unknown TruthBench disposition: {value!r}") from exc + + +@dataclass(frozen=True) +class GoldCell: + cell_id: str + fixture_version: str + domain: str + row_id: str + column: str + disposition: Disposition + expected_output: TypedValue | None = None + family: str | None = None + sensitive: bool = False + canary_id: str | None = None + schema_version: int = field(default=1, init=False) + + @classmethod + def create( + cls, + fixture_version: str, + domain: str, + row_id: str, + column: str, + disposition: Disposition | str, + *, + expected_output: Any = UNSET, + expected_dtype: Any = None, + family: str | None = None, + sensitive: bool = False, + canary_id: str | None = None, + digest_key: bytes = _TEST_DIGEST_KEY, + ) -> GoldCell: + identifiers = (fixture_version, domain, row_id, column) + if any(not isinstance(item, str) or not item for item in identifiers): + raise ValueError("cell identity components must be non-empty strings") + typed_output = None + if expected_output is not UNSET: + typed_output = encode_typed( + expected_output, + dtype=expected_dtype, + sensitive=sensitive, + digest_key=digest_key if sensitive else None, + ) + return cls( + cell_id=":".join(identifiers), + fixture_version=fixture_version, + domain=domain, + row_id=row_id, + column=column, + disposition=_disposition(disposition), + expected_output=typed_output, + family=family, + sensitive=sensitive, + canary_id=canary_id, + ) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "cell_id": self.cell_id, + "fixture_version": self.fixture_version, + "domain": self.domain, + "row_id": self.row_id, + "column": self.column, + "disposition": self.disposition.value, + "expected_output": ( + None if self.expected_output is None else self.expected_output.to_dict() + ), + "family": self.family, + "sensitive": self.sensitive, + "canary_id": self.canary_id, + } + + +@dataclass(frozen=True) +class CaseExpectation: + case_id: str + fixture_version: str + domain: str + kind: str + name: str + disposition: Disposition + expected: TypedValue | None = None + family: str | None = None + sensitive: bool = False + schema_version: int = field(default=1, init=False) + + @classmethod + def create( + cls, + fixture_version: str, + domain: str, + kind: str, + name: str, + disposition: Disposition | str, + *, + expected: Any = UNSET, + expected_dtype: Any = None, + family: str | None = None, + sensitive: bool = False, + digest_key: bytes = _TEST_DIGEST_KEY, + ) -> CaseExpectation: + identifiers = (fixture_version, domain, kind, name) + if any(not isinstance(item, str) or not item for item in identifiers): + raise ValueError("case identity components must be non-empty strings") + typed_expected = None + if expected is not UNSET: + typed_expected = encode_typed( + expected, + dtype=expected_dtype, + sensitive=sensitive, + digest_key=digest_key if sensitive else None, + ) + return cls( + case_id=":".join(identifiers), + fixture_version=fixture_version, + domain=domain, + kind=kind, + name=name, + disposition=_disposition(disposition), + expected=typed_expected, + family=family, + sensitive=sensitive, + ) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "case_id": self.case_id, + "fixture_version": self.fixture_version, + "domain": self.domain, + "kind": self.kind, + "name": self.name, + "disposition": self.disposition.value, + "expected": None if self.expected is None else self.expected.to_dict(), + "family": self.family, + "sensitive": self.sensitive, + } + + +@dataclass(frozen=True) +class DecisionRecord: + record_id: str + run_id: str + fixture_id: str + case_id: str | None + cell_id: str + domain: str + row_id: str + column: str + surface: str + repeat: int + expected_disposition: Disposition + actual_disposition: Disposition | None + input: TypedValue + expected_output: TypedValue | None + actual_output: TypedValue | None + confidence: float | None = None + risk: str | None = None + status: str | None = None + rule_id: str | None = None + rationale: str | None = None + evidence_kinds: tuple[str, ...] | None = None + mutated: bool | None = None + detected: bool | None = None + quarantined: bool | None = None + human_review: bool | None = None + audit_required: bool | None = None + audit_complete: bool | None = None + audit_ids: tuple[str, ...] | None = None + trust_before: float | None = None + trust_after: float | None = None + trust_delta: float | None = None + requested_backend: str | None = None + actual_backend: str | None = None + fallback_events: tuple[str, ...] | None = None + backend_differences: tuple[str, ...] | None = None + normalized_decision_hash: str | None = None + repeat_hash: str | None = None + repeat_consistent: bool | None = None + schema_version: int = field(default=1, init=False) + + @classmethod + def for_test(cls, *, cell: GoldCell, input_value: Any) -> DecisionRecord: + typed_input = encode_typed( + input_value, + sensitive=cell.sensitive, + digest_key=_TEST_DIGEST_KEY if cell.sensitive else None, + ) + return cls( + record_id=f"test:{cell.cell_id}", + run_id="test", + fixture_id=f"{cell.fixture_version}:{cell.domain}", + case_id=None, + cell_id=cell.cell_id, + domain=cell.domain, + row_id=cell.row_id, + column=cell.column, + surface="test", + repeat=0, + expected_disposition=cell.disposition, + actual_disposition=None, + input=typed_input, + expected_output=cell.expected_output, + actual_output=None, + ) + + @staticmethod + def _typed(value: TypedValue | None) -> dict[str, JsonValue] | None: + return None if value is None else value.to_dict() + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "record_id": self.record_id, + "run_id": self.run_id, + "fixture_id": self.fixture_id, + "case_id": self.case_id, + "cell_id": self.cell_id, + "domain": self.domain, + "row_id": self.row_id, + "column": self.column, + "surface": self.surface, + "repeat": self.repeat, + "expected_disposition": self.expected_disposition.value, + "actual_disposition": ( + None if self.actual_disposition is None else self.actual_disposition.value + ), + "input": self.input.to_dict(), + "input_type": self.input.type_label, + "expected_output": self._typed(self.expected_output), + "expected_output_type": ( + None if self.expected_output is None else self.expected_output.type_label + ), + "actual_output": self._typed(self.actual_output), + "actual_output_type": ( + None if self.actual_output is None else self.actual_output.type_label + ), + "confidence": self.confidence, + "risk": self.risk, + "status": self.status, + "rule_id": self.rule_id, + "rationale": self.rationale, + "evidence_kinds": ( + None if self.evidence_kinds is None else list(self.evidence_kinds) + ), + "mutated": self.mutated, + "detected": self.detected, + "quarantined": self.quarantined, + "human_review": self.human_review, + "audit_required": self.audit_required, + "audit_complete": self.audit_complete, + "audit_ids": None if self.audit_ids is None else list(self.audit_ids), + "trust_before": self.trust_before, + "trust_after": self.trust_after, + "trust_delta": self.trust_delta, + "requested_backend": self.requested_backend, + "actual_backend": self.actual_backend, + "fallback_events": ( + None if self.fallback_events is None else list(self.fallback_events) + ), + "backend_differences": ( + None if self.backend_differences is None else list(self.backend_differences) + ), + "normalized_decision_hash": self.normalized_decision_hash, + "repeat_hash": self.repeat_hash, + "repeat_consistent": self.repeat_consistent, + } + + +@dataclass(frozen=True) +class GateResult: + name: str + passed: bool + failures: tuple[str, ...] = () + schema_version: int = field(default=1, init=False) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "name": self.name, + "passed": self.passed, + "failure_count": len(self.failures), + "failures": list(self.failures), + } + + +@dataclass(frozen=True) +class RunResult: + run_id: str + profile: str + fixture_hashes: tuple[tuple[str, str], ...] + required_backends: tuple[str, ...] + records: tuple[DecisionRecord, ...] + gates: tuple[GateResult, ...] + summary: tuple[tuple[str, JsonValue], ...] + environment: tuple[tuple[str, JsonValue], ...] + schema_version: int = field(default=1, init=False) + + def to_dict(self) -> dict[str, JsonValue]: + return { + "schema_version": self.schema_version, + "run_id": self.run_id, + "profile": self.profile, + "fixture_hashes": dict(self.fixture_hashes), + "required_backends": list(self.required_backends), + "records": [record.to_dict() for record in self.records], + "gates": [gate.to_dict() for gate in self.gates], + "summary": dict(self.summary), + "environment": dict(self.environment), + } diff --git a/tests/truthbench/test_models_exact.py b/tests/truthbench/test_models_exact.py new file mode 100644 index 0000000..766581a --- /dev/null +++ b/tests/truthbench/test_models_exact.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import json +from dataclasses import FrozenInstanceError +from datetime import datetime, timezone + +import numpy as np +import pandas as pd +import pytest +from benchmarks.truthbench import ( + CaseExpectation, + DecisionRecord, + Disposition, + GateResult, + GoldCell, + RunResult, + TypedValue, + canonical_json, + encode_typed, + exact_equal, + stable_digest, +) + + +def test_disposition_contract_has_exactly_four_string_values() -> None: + assert [(item.name, item.value) for item in Disposition] == [ + ("PRESERVE", "preserve"), + ("REPAIR", "repair"), + ("FLAG", "flag"), + ("REVIEW", "review"), + ] + assert isinstance(Disposition.PRESERVE, str) + + +def test_gold_cell_ids_are_stable_and_match_the_public_contract() -> None: + first = GoldCell.create("v1", "crm", "r7", "notes", "flag", sensitive=True) + second = GoldCell.create("v1", "crm", "r7", "notes", Disposition.FLAG, sensitive=True) + + assert first == second + assert first.cell_id == "v1:crm:r7:notes" + assert GoldCell.create("v1", "crm", "r7", "email", "flag").cell_id != first.cell_id + + +def test_exact_values_do_not_use_gauntlet_canonicalization() -> None: + assert not exact_equal("402.10", 402.1) + assert not exact_equal(" AAPL", "AAPL") + assert not exact_equal("AAPL", "aapl") + assert exact_equal(pd.NA, pd.NA) + + +def test_exact_values_preserve_scalar_type_and_missing_kind() -> None: + assert not exact_equal(402.1, np.float64(402.1)) + assert not exact_equal(None, pd.NA) + assert not exact_equal(None, float("nan")) + assert not exact_equal(pd.NA, float("nan")) + assert exact_equal(float("nan"), float("nan")) + assert exact_equal(np.float64(402.1), np.float64(402.1)) + + +def test_exact_values_preserve_unicode_timestamp_and_identifier_representation() -> None: + assert not exact_equal("Café", "Cafe\u0301") + assert not exact_equal("00127", 127) + assert exact_equal("00127", "00127") + + utc = pd.Timestamp("2026-01-15T12:00:00+00:00") + kolkata = pd.Timestamp("2026-01-15T17:30:00+05:30") + assert not exact_equal(utc, kolkata) + assert not exact_equal(utc, datetime(2026, 1, 15, 12, tzinfo=timezone.utc)) + + +def test_dtype_metadata_distinguishes_string_and_categorical_scalars() -> None: + string_dtype = pd.StringDtype(storage="python") + category_dtype = pd.CategoricalDtype(categories=["A", "B"], ordered=True) + + assert encode_typed("A", dtype=string_dtype) != encode_typed("A", dtype=category_dtype) + assert exact_equal("A", "A", left_dtype=string_dtype, right_dtype=string_dtype) + assert not exact_equal("A", "A", left_dtype=string_dtype, right_dtype=object) + + +def test_sensitive_record_never_serializes_raw_value() -> None: + cell = GoldCell.create("v1", "crm", "r7", "notes", "flag", sensitive=True) + record = DecisionRecord.for_test( + cell=cell, + input_value="tb.person+7@example.invalid", + ) + + payload = record.to_dict() + + assert "tb.person+7@example.invalid" not in json.dumps(payload) + assert payload["input"]["display"] == "[REDACTED]" + assert payload["input"]["value"] is None + assert payload["input"]["digest"] + + +def test_decision_record_serializes_every_normalized_dimension() -> None: + cell = GoldCell.create("v1", "finance", "r1", "price", "repair", expected_output=402.1) + payload = DecisionRecord.for_test(cell=cell, input_value="402.10").to_dict() + + required = { + "expected_disposition", + "actual_disposition", + "input", + "input_type", + "expected_output", + "expected_output_type", + "actual_output", + "actual_output_type", + "confidence", + "rationale", + "audit_required", + "audit_complete", + "audit_ids", + "trust_before", + "trust_after", + "trust_delta", + "requested_backend", + "actual_backend", + "fallback_events", + "repeat_hash", + "repeat_consistent", + } + assert required <= payload.keys() + populated = { + "expected_disposition", + "input", + "input_type", + "expected_output", + "expected_output_type", + } + assert all(payload[key] is None for key in required if key not in populated) + + +def test_oracle_and_result_models_are_frozen_and_json_safe() -> None: + cell = GoldCell.create("v1", "crm", "r7", "notes", "review") + case = CaseExpectation.create("v1", "crm", "row", "duplicate-r7", "review") + record = DecisionRecord.for_test(cell=cell, input_value="ordinary text") + gate = GateResult(name="valid-value-corruption", passed=True) + run = RunResult( + run_id="run-1", + profile="release", + fixture_hashes=(("crm", "abc123"),), + required_backends=("pandas",), + records=(record,), + gates=(gate,), + summary=(("overall_passed", True), ("records", 1)), + environment=(("python", "3.9+"),), + ) + + for value in (encode_typed("x"), cell, case, record, gate, run): + with pytest.raises(FrozenInstanceError): + value.schema_version = 2 # type: ignore[misc] + + payload = run.to_dict() + assert json.loads(canonical_json(payload)) == payload + serialized = ( + cell.to_dict(), + case.to_dict(), + record.to_dict(), + gate.to_dict(), + payload, + ) + assert all(item["schema_version"] == 1 for item in serialized) + + +def test_canonical_json_is_stable_and_rejects_unsafe_values() -> None: + assert canonical_json({"b": 2, "a": [True, None]}) == '{"a":[true,null],"b":2}' + assert stable_digest("AAPL", key=b"run-one") == stable_digest("AAPL", key=b"run-one") + assert stable_digest("AAPL", key=b"run-one") != stable_digest("AAPL", key=b"run-two") + + for value in (float("nan"), float("inf"), float("-inf")): + with pytest.raises(ValueError, match="finite"): + canonical_json({"unsafe": value}) + with pytest.raises(TypeError, match="unsupported"): + canonical_json({"unsafe": object()}) + + +def test_typed_values_round_trip_without_non_finite_json_numbers() -> None: + values = [ + "402.10", + 402.1, + np.float64(402.1), + None, + pd.NA, + float("nan"), + "Café", + "Cafe\u0301", + pd.Timestamp("2026-01-15T12:00:00+00:00"), + "00127", + ] + + for value in values: + typed = encode_typed(value) + payload = typed.to_dict() + assert TypedValue.from_dict(json.loads(canonical_json(payload))) == typed From 4117574d932be8e8928e262aa25391b603f3d3b6 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 11:15:13 +0530 Subject: [PATCH 05/40] fix: harden TruthBench typed invariants --- benchmarks/truthbench/exact.py | 6 +- benchmarks/truthbench/models.py | 38 +++++++-- tests/truthbench/test_models_exact.py | 110 +++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 19 deletions(-) diff --git a/benchmarks/truthbench/exact.py b/benchmarks/truthbench/exact.py index 966a173..4246ae4 100644 --- a/benchmarks/truthbench/exact.py +++ b/benchmarks/truthbench/exact.py @@ -130,7 +130,11 @@ def _encode_scalar(value: Any) -> tuple[str, JsonValue, str]: if isinstance(value, pd.Timestamp): rendered = value.isoformat() - return "pandas.Timestamp", rendered, rendered + encoded = { + "iso": rendered, + "timezone": None if value.tz is None else str(value.tz), + } + return "pandas.Timestamp", encoded, rendered if isinstance(value, datetime): rendered = value.isoformat() return "python.datetime", rendered, rendered diff --git a/benchmarks/truthbench/models.py b/benchmarks/truthbench/models.py index 0ff5132..2815667 100644 --- a/benchmarks/truthbench/models.py +++ b/benchmarks/truthbench/models.py @@ -29,6 +29,14 @@ def _disposition(value: Disposition | str) -> Disposition: raise ValueError(f"unknown TruthBench disposition: {value!r}") from exc +def _stable_id(*components: str) -> str: + if any(not isinstance(item, str) or not item for item in components): + raise ValueError("stable ID components must be non-empty strings") + if any(":" in item for item in components): + raise ValueError("stable ID components must not contain a colon") + return ":".join(components) + + @dataclass(frozen=True) class GoldCell: cell_id: str @@ -59,9 +67,7 @@ def create( canary_id: str | None = None, digest_key: bytes = _TEST_DIGEST_KEY, ) -> GoldCell: - identifiers = (fixture_version, domain, row_id, column) - if any(not isinstance(item, str) or not item for item in identifiers): - raise ValueError("cell identity components must be non-empty strings") + cell_id = _stable_id(fixture_version, domain, row_id, column) typed_output = None if expected_output is not UNSET: typed_output = encode_typed( @@ -71,7 +77,7 @@ def create( digest_key=digest_key if sensitive else None, ) return cls( - cell_id=":".join(identifiers), + cell_id=cell_id, fixture_version=fixture_version, domain=domain, row_id=row_id, @@ -129,9 +135,7 @@ def create( sensitive: bool = False, digest_key: bytes = _TEST_DIGEST_KEY, ) -> CaseExpectation: - identifiers = (fixture_version, domain, kind, name) - if any(not isinstance(item, str) or not item for item in identifiers): - raise ValueError("case identity components must be non-empty strings") + case_id = _stable_id(fixture_version, domain, kind, name) typed_expected = None if expected is not UNSET: typed_expected = encode_typed( @@ -141,7 +145,7 @@ def create( digest_key=digest_key if sensitive else None, ) return cls( - case_id=":".join(identifiers), + case_id=case_id, fixture_version=fixture_version, domain=domain, kind=kind, @@ -181,6 +185,7 @@ class DecisionRecord: repeat: int expected_disposition: Disposition actual_disposition: Disposition | None + sensitive: bool input: TypedValue expected_output: TypedValue | None actual_output: TypedValue | None @@ -209,6 +214,21 @@ class DecisionRecord: repeat_consistent: bool | None = None schema_version: int = field(default=1, init=False) + def __post_init__(self) -> None: + if not self.sensitive: + return + for name in ("input", "expected_output", "actual_output"): + value = getattr(self, name) + if value is not None and not ( + value.redacted + and value.value is None + and value.display == "[REDACTED]" + and bool(value.digest) + ): + raise ValueError( + f"sensitive DecisionRecord {name} must be a redacted TypedValue" + ) + @classmethod def for_test(cls, *, cell: GoldCell, input_value: Any) -> DecisionRecord: typed_input = encode_typed( @@ -232,6 +252,7 @@ def for_test(cls, *, cell: GoldCell, input_value: Any) -> DecisionRecord: input=typed_input, expected_output=cell.expected_output, actual_output=None, + sensitive=cell.sensitive, ) @staticmethod @@ -255,6 +276,7 @@ def to_dict(self) -> dict[str, JsonValue]: "actual_disposition": ( None if self.actual_disposition is None else self.actual_disposition.value ), + "sensitive": self.sensitive, "input": self.input.to_dict(), "input_type": self.input.type_label, "expected_output": self._typed(self.expected_output), diff --git a/tests/truthbench/test_models_exact.py b/tests/truthbench/test_models_exact.py index 766581a..8efc07c 100644 --- a/tests/truthbench/test_models_exact.py +++ b/tests/truthbench/test_models_exact.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from dataclasses import FrozenInstanceError +from dataclasses import FrozenInstanceError, fields, replace from datetime import datetime, timezone import numpy as np @@ -41,6 +41,19 @@ def test_gold_cell_ids_are_stable_and_match_the_public_contract() -> None: assert GoldCell.create("v1", "crm", "r7", "email", "flag").cell_id != first.cell_id +@pytest.mark.parametrize("component", range(4)) +def test_stable_id_components_reject_colons_that_could_collide(component: int) -> None: + cell_parts = ["v1", "crm", "r7", "notes"] + cell_parts[component] = f"{cell_parts[component]}:ambiguous" + with pytest.raises(ValueError, match="colon"): + GoldCell.create(*cell_parts, "flag") + + case_parts = ["v1", "crm", "row", "duplicate-r7"] + case_parts[component] = f"{case_parts[component]}:ambiguous" + with pytest.raises(ValueError, match="colon"): + CaseExpectation.create(*case_parts, "review") + + def test_exact_values_do_not_use_gauntlet_canonicalization() -> None: assert not exact_equal("402.10", 402.1) assert not exact_equal(" AAPL", "AAPL") @@ -68,6 +81,14 @@ def test_exact_values_preserve_unicode_timestamp_and_identifier_representation() assert not exact_equal(utc, datetime(2026, 1, 15, 12, tzinfo=timezone.utc)) +def test_exact_timestamps_preserve_timezone_identity_with_the_same_offset() -> None: + utc = pd.Timestamp("2026-01-15T12:00:00", tz="UTC") + london = pd.Timestamp("2026-01-15T12:00:00", tz="Europe/London") + + assert utc.isoformat() == london.isoformat() + assert not exact_equal(utc, london) + + def test_dtype_metadata_distinguishes_string_and_categorical_scalars() -> None: string_dtype = pd.StringDtype(storage="python") category_dtype = pd.CategoricalDtype(categories=["A", "B"], ordered=True) @@ -92,13 +113,51 @@ def test_sensitive_record_never_serializes_raw_value() -> None: assert payload["input"]["digest"] +@pytest.mark.parametrize("field", ["input", "expected_output", "actual_output"]) +def test_direct_sensitive_record_rejects_unredacted_typed_values(field: str) -> None: + cell = GoldCell.create("v1", "crm", "r7", "notes", "flag", sensitive=True) + record = DecisionRecord.for_test(cell=cell, input_value="ordinary text") + + with pytest.raises(ValueError, match=field): + replace( + record, + sensitive=True, + **{field: encode_typed("tb.person+7@example.invalid")}, + ) + + +def test_direct_record_requires_explicit_sensitive_classification() -> None: + cell = GoldCell.create("v1", "crm", "r7", "notes", "flag") + record = DecisionRecord.for_test(cell=cell, input_value="ordinary text") + constructor = { + item.name: getattr(record, item.name) + for item in fields(record) + if item.init and item.name != "sensitive" + } + + with pytest.raises(TypeError, match="sensitive"): + DecisionRecord(**constructor) + + def test_decision_record_serializes_every_normalized_dimension() -> None: cell = GoldCell.create("v1", "finance", "r1", "price", "repair", expected_output=402.1) payload = DecisionRecord.for_test(cell=cell, input_value="402.10").to_dict() - required = { + assert set(payload) == { + "schema_version", + "record_id", + "run_id", + "fixture_id", + "case_id", + "cell_id", + "domain", + "row_id", + "column", + "surface", + "repeat", "expected_disposition", "actual_disposition", + "sensitive", "input", "input_type", "expected_output", @@ -106,7 +165,15 @@ def test_decision_record_serializes_every_normalized_dimension() -> None: "actual_output", "actual_output_type", "confidence", + "risk", + "status", + "rule_id", "rationale", + "evidence_kinds", + "mutated", + "detected", + "quarantined", + "human_review", "audit_required", "audit_complete", "audit_ids", @@ -116,18 +183,41 @@ def test_decision_record_serializes_every_normalized_dimension() -> None: "requested_backend", "actual_backend", "fallback_events", + "backend_differences", + "normalized_decision_hash", "repeat_hash", "repeat_consistent", } - assert required <= payload.keys() - populated = { - "expected_disposition", - "input", - "input_type", - "expected_output", - "expected_output_type", + nullable = { + "case_id", + "actual_disposition", + "actual_output", + "actual_output_type", + "confidence", + "risk", + "status", + "rule_id", + "rationale", + "evidence_kinds", + "mutated", + "detected", + "quarantined", + "human_review", + "audit_required", + "audit_complete", + "audit_ids", + "trust_before", + "trust_after", + "trust_delta", + "requested_backend", + "actual_backend", + "fallback_events", + "backend_differences", + "normalized_decision_hash", + "repeat_hash", + "repeat_consistent", } - assert all(payload[key] is None for key in required if key not in populated) + assert all(payload[key] is None for key in nullable) def test_oracle_and_result_models_are_frozen_and_json_safe() -> None: From 862a8aea8ab6e53b753c429e6cab038ad3d3a3d7 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 11:19:43 +0530 Subject: [PATCH 06/40] fix: preserve Python timezone identity --- benchmarks/truthbench/exact.py | 17 ++++++++++++++--- tests/truthbench/test_models_exact.py | 23 ++++++++++++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/benchmarks/truthbench/exact.py b/benchmarks/truthbench/exact.py index 4246ae4..98dacd3 100644 --- a/benchmarks/truthbench/exact.py +++ b/benchmarks/truthbench/exact.py @@ -78,6 +78,15 @@ def _dtype_name(dtype: Any, *, sensitive: bool) -> str | None: return str(normalized) +def _timezone_identity(value: Any) -> str | None: + if value is None: + return None + key = getattr(value, "key", None) + if isinstance(key, str): + return key + return str(value) + + def _encode_scalar(value: Any) -> tuple[str, JsonValue, str]: if value is pd.NA: return "pandas.NA", None, "" @@ -132,18 +141,20 @@ def _encode_scalar(value: Any) -> tuple[str, JsonValue, str]: rendered = value.isoformat() encoded = { "iso": rendered, - "timezone": None if value.tz is None else str(value.tz), + "timezone": _timezone_identity(value.tz), } return "pandas.Timestamp", encoded, rendered if isinstance(value, datetime): rendered = value.isoformat() - return "python.datetime", rendered, rendered + encoded = {"iso": rendered, "timezone": _timezone_identity(value.tzinfo)} + return "python.datetime", encoded, rendered if isinstance(value, date): rendered = value.isoformat() return "python.date", rendered, rendered if isinstance(value, time): rendered = value.isoformat() - return "python.time", rendered, rendered + encoded = {"iso": rendered, "timezone": _timezone_identity(value.tzinfo)} + return "python.time", encoded, rendered if isinstance(value, pd.Timedelta): rendered = value.isoformat() diff --git a/tests/truthbench/test_models_exact.py b/tests/truthbench/test_models_exact.py index 8efc07c..481c89a 100644 --- a/tests/truthbench/test_models_exact.py +++ b/tests/truthbench/test_models_exact.py @@ -2,7 +2,8 @@ import json from dataclasses import FrozenInstanceError, fields, replace -from datetime import datetime, timezone +from datetime import datetime, time, timezone +from zoneinfo import ZoneInfo import numpy as np import pandas as pd @@ -89,6 +90,26 @@ def test_exact_timestamps_preserve_timezone_identity_with_the_same_offset() -> N assert not exact_equal(utc, london) +def test_exact_python_datetimes_preserve_same_offset_timezone_identity() -> None: + utc = datetime(2026, 1, 15, 12, tzinfo=ZoneInfo("UTC")) + london = datetime(2026, 1, 15, 12, tzinfo=ZoneInfo("Europe/London")) + + assert utc.isoformat() == london.isoformat() + assert not exact_equal(utc, london) + + +def test_exact_python_times_preserve_same_offset_timezone_identity() -> None: + utc = time(12, tzinfo=ZoneInfo("UTC")) + london = time(12, tzinfo=ZoneInfo("Europe/London")) + + assert encode_typed(utc).value == {"iso": utc.isoformat(), "timezone": "UTC"} + assert encode_typed(london).value == { + "iso": london.isoformat(), + "timezone": "Europe/London", + } + assert not exact_equal(utc, london) + + def test_dtype_metadata_distinguishes_string_and_categorical_scalars() -> None: string_dtype = pd.StringDtype(storage="python") category_dtype = pd.CategoricalDtype(categories=["A", "B"], ordered=True) From 963fd253d8cc94f9ec7a26926bbd7d7264b8fb11 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 11:34:05 +0530 Subject: [PATCH 07/40] feat: validate TruthBench result integrity --- benchmarks/truthbench/schema.py | 244 ++++++++++++++++++++++++++++++++ tests/truthbench/test_schema.py | 173 ++++++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 benchmarks/truthbench/schema.py create mode 100644 tests/truthbench/test_schema.py diff --git a/benchmarks/truthbench/schema.py b/benchmarks/truthbench/schema.py new file mode 100644 index 0000000..b318080 --- /dev/null +++ b/benchmarks/truthbench/schema.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import Any + +import jsonschema + + +class TruthBenchSchemaError(ValueError): + """A serialized TruthBench run violates its schema or integrity contract.""" + + +_NULLABLE_STRING = {"type": ["string", "null"]} +_NULLABLE_BOOLEAN = {"type": ["boolean", "null"]} +_NULLABLE_NUMBER = {"type": ["number", "null"]} +_NULLABLE_STRING_ARRAY = { + "oneOf": [ + {"type": "null"}, + {"type": "array", "items": {"type": "string"}}, + ] +} + +_TYPED_VALUE_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": [ + "schema_version", + "type", + "dtype", + "value", + "display", + "digest", + "redacted", + ], + "properties": { + "schema_version": {"const": 1}, + "type": {"type": "string", "minLength": 1}, + "dtype": _NULLABLE_STRING, + "value": {}, + "display": {"type": "string"}, + "digest": _NULLABLE_STRING, + "redacted": {"type": "boolean"}, + }, +} + +_RECORD_PROPERTIES: dict[str, Any] = { + "schema_version": {"const": 1}, + "record_id": {"type": "string", "minLength": 1}, + "run_id": {"type": "string", "minLength": 1}, + "fixture_id": {"type": "string", "minLength": 1}, + "case_id": _NULLABLE_STRING, + "cell_id": {"type": "string", "minLength": 1}, + "domain": {"type": "string", "minLength": 1}, + "row_id": {"type": "string", "minLength": 1}, + "column": {"type": "string", "minLength": 1}, + "surface": {"type": "string", "minLength": 1}, + "repeat": {"type": "integer", "minimum": 0}, + "expected_disposition": { + "enum": ["preserve", "repair", "flag", "review"], + }, + "actual_disposition": { + "enum": ["preserve", "repair", "flag", "review", None], + }, + "sensitive": {"type": "boolean"}, + "input": _TYPED_VALUE_SCHEMA, + "input_type": {"type": "string", "minLength": 1}, + "expected_output": { + "oneOf": [{"type": "null"}, _TYPED_VALUE_SCHEMA], + }, + "expected_output_type": _NULLABLE_STRING, + "actual_output": { + "oneOf": [{"type": "null"}, _TYPED_VALUE_SCHEMA], + }, + "actual_output_type": _NULLABLE_STRING, + "confidence": _NULLABLE_NUMBER, + "risk": _NULLABLE_STRING, + "status": _NULLABLE_STRING, + "rule_id": _NULLABLE_STRING, + "rationale": _NULLABLE_STRING, + "evidence_kinds": _NULLABLE_STRING_ARRAY, + "mutated": _NULLABLE_BOOLEAN, + "detected": _NULLABLE_BOOLEAN, + "quarantined": _NULLABLE_BOOLEAN, + "human_review": _NULLABLE_BOOLEAN, + "audit_required": _NULLABLE_BOOLEAN, + "audit_complete": _NULLABLE_BOOLEAN, + "audit_ids": _NULLABLE_STRING_ARRAY, + "trust_before": _NULLABLE_NUMBER, + "trust_after": _NULLABLE_NUMBER, + "trust_delta": _NULLABLE_NUMBER, + "requested_backend": _NULLABLE_STRING, + "actual_backend": _NULLABLE_STRING, + "fallback_events": _NULLABLE_STRING_ARRAY, + "backend_differences": _NULLABLE_STRING_ARRAY, + "normalized_decision_hash": _NULLABLE_STRING, + "repeat_hash": _NULLABLE_STRING, + "repeat_consistent": _NULLABLE_BOOLEAN, +} + +_RECORD_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": list(_RECORD_PROPERTIES), + "properties": _RECORD_PROPERTIES, +} + +_GATE_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": ["schema_version", "name", "passed", "failure_count", "failures"], + "properties": { + "schema_version": {"const": 1}, + "name": {"type": "string", "minLength": 1}, + "passed": {"type": "boolean"}, + "failure_count": {"type": "integer", "minimum": 0}, + "failures": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + }, + }, +} + +_ENVIRONMENT_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": ["python"], + "properties": { + "python": {"type": "string", "minLength": 1}, + }, +} + +RESULT_SCHEMA: dict[str, Any] = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": False, + "required": [ + "schema_version", + "run_id", + "profile", + "fixture_hashes", + "required_backends", + "records", + "gates", + "summary", + "environment", + ], + "properties": { + "schema_version": {"const": 1}, + "run_id": {"type": "string", "minLength": 1}, + "profile": {"enum": ["release", "extended"]}, + "fixture_hashes": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"type": "string", "minLength": 1}, + }, + "required_backends": { + "type": "array", + "minItems": 1, + "uniqueItems": True, + "items": {"type": "string", "minLength": 1}, + }, + "records": { + "type": "array", + "minItems": 1, + "items": _RECORD_SCHEMA, + }, + "gates": { + "type": "array", + "minItems": 1, + "items": _GATE_SCHEMA, + }, + "summary": { + "type": "object", + "required": ["records", "overall_passed"], + "properties": { + "records": {"type": "integer", "minimum": 0}, + "overall_passed": {"type": "boolean"}, + }, + }, + "environment": _ENVIRONMENT_SCHEMA, + }, +} + +_VALIDATOR = jsonschema.Draft202012Validator(RESULT_SCHEMA) +_SCORE_FIELDS = ("confidence", "trust_before", "trust_after", "trust_delta") + + +def _schema_path(error: jsonschema.ValidationError) -> str: + path = "$" + for component in error.absolute_path: + if isinstance(component, int): + path += f"[{component}]" + else: + path += f".{component}" + return path + + +def _validate_schema(payload: Mapping[str, Any]) -> None: + try: + _VALIDATOR.validate(dict(payload)) + except jsonschema.ValidationError as exc: + raise TruthBenchSchemaError( + f"schema validation failed at {_schema_path(exc)}: {exc.message}" + ) from exc + + +def validate_run(payload: Mapping[str, Any]) -> None: + """Validate a serialized ``RunResult`` and its aggregate integrity.""" + + _validate_schema(payload) + + records = payload["records"] + for index, record in enumerate(records): + for field in _SCORE_FIELDS: + value = record[field] + if value is not None and not math.isfinite(value): + raise TruthBenchSchemaError(f"record {index} {field} must be a finite number") + + ids = [record["record_id"] for record in records] + if len(ids) != len(set(ids)): + raise TruthBenchSchemaError("duplicate decision record id") + + fixture_hashes = payload["fixture_hashes"] + for record in records: + domain = record["domain"] + fixture_domain = record["fixture_id"].rsplit(":", 1)[-1] + if domain != fixture_domain or domain not in fixture_hashes: + raise TruthBenchSchemaError( + f"decision record fixture hash is missing for {record['fixture_id']}" + ) + + if payload["summary"]["records"] != len(ids): + raise TruthBenchSchemaError("record aggregate does not match records") + + for gate in payload["gates"]: + if gate["failure_count"] != len(gate["failures"]): + raise TruthBenchSchemaError( + f"gate failure aggregate does not match failures for {gate['name']}" + ) + + passed = all(gate["passed"] for gate in payload["gates"]) + if payload["summary"]["overall_passed"] is not passed: + raise TruthBenchSchemaError("overall gate claim is inconsistent") diff --git a/tests/truthbench/test_schema.py b/tests/truthbench/test_schema.py new file mode 100644 index 0000000..57b8e37 --- /dev/null +++ b/tests/truthbench/test_schema.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from copy import deepcopy +from types import MappingProxyType +from typing import Any + +import pytest +from benchmarks.truthbench.models import ( + DecisionRecord, + GateResult, + GoldCell, + RunResult, +) +from benchmarks.truthbench.schema import TruthBenchSchemaError, validate_run + + +def _valid_payload() -> dict[str, Any]: + cell = GoldCell.create("v1", "crm", "r7", "notes", "review") + record = DecisionRecord.for_test(cell=cell, input_value="ordinary text") + run = RunResult( + run_id="run-1", + profile="release", + fixture_hashes=(("crm", "abc123"),), + required_backends=("pandas",), + records=(record,), + gates=(GateResult(name="valid-value-corruption", passed=True),), + summary=(("overall_passed", True), ("records", 1)), + environment=(("python", "3.9+"),), + ) + return run.to_dict() + + +def test_valid_serialized_run_passes_schema_and_integrity_validation() -> None: + validate_run(_valid_payload()) + + +def test_validator_accepts_the_declared_mapping_interface() -> None: + validate_run(MappingProxyType(_valid_payload())) + + +@pytest.mark.parametrize( + "path", + [ + (), + ("records", 0), + ("gates", 0), + ("records", 0, "input"), + ], +) +def test_unknown_schema_versions_are_rejected(path: tuple[str | int, ...]) -> None: + payload = _valid_payload() + target: Any = payload + for component in path: + target = target[component] + target["schema_version"] = 2 + + with pytest.raises(TruthBenchSchemaError, match="schema"): + validate_run(payload) + + +@pytest.mark.parametrize("field", ["fixture_hashes", "required_backends", "gates"]) +def test_required_run_components_cannot_be_absent_or_empty(field: str) -> None: + for missing in (True, False): + payload = _valid_payload() + if missing: + del payload[field] + else: + payload[field] = {} if field == "fixture_hashes" else [] + + with pytest.raises(TruthBenchSchemaError): + validate_run(payload) + + +def test_run_without_decision_records_is_partial_and_rejected() -> None: + payload = _valid_payload() + payload["records"] = [] + payload["summary"]["records"] = 0 + + with pytest.raises(TruthBenchSchemaError): + validate_run(payload) + + +def test_duplicate_record_ids_are_rejected_precisely() -> None: + payload = _valid_payload() + payload["records"].append(deepcopy(payload["records"][0])) + payload["summary"]["records"] = 2 + + with pytest.raises(TruthBenchSchemaError, match="duplicate decision record id"): + validate_run(payload) + + +@pytest.mark.parametrize( + "field", + ["confidence", "trust_before", "trust_after", "trust_delta"], +) +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_non_finite_record_scores_are_rejected(field: str, value: float) -> None: + payload = _valid_payload() + payload["records"][0][field] = value + + with pytest.raises(TruthBenchSchemaError, match="finite"): + validate_run(payload) + + +def test_each_record_must_have_a_matching_fixture_hash() -> None: + payload = _valid_payload() + payload["fixture_hashes"] = {"finance": "def456"} + + with pytest.raises(TruthBenchSchemaError, match="fixture hash"): + validate_run(payload) + + +def test_record_aggregate_must_match_record_count() -> None: + payload = _valid_payload() + payload["summary"]["records"] = 9 + + with pytest.raises( + TruthBenchSchemaError, + match="record aggregate does not match records", + ): + validate_run(payload) + + +def test_gate_failure_aggregate_must_match_failures() -> None: + payload = _valid_payload() + payload["gates"][0]["failure_count"] = 1 + + with pytest.raises(TruthBenchSchemaError, match="gate failure aggregate"): + validate_run(payload) + + +def test_overall_passed_claim_cannot_hide_a_failed_gate() -> None: + payload = _valid_payload() + payload["gates"][0] = GateResult( + name="valid-value-corruption", + passed=False, + failures=("record changed",), + ).to_dict() + + with pytest.raises(TruthBenchSchemaError, match="overall gate claim is inconsistent"): + validate_run(payload) + + +@pytest.mark.parametrize( + ("path", "extra"), + [ + ((), "unexpected_result"), + (("records", 0), "unexpected_record"), + (("gates", 0), "unexpected_gate"), + (("environment",), "unexpected_environment"), + ], +) +def test_closed_schema_levels_reject_unknown_properties( + path: tuple[str | int, ...], + extra: str, +) -> None: + payload = _valid_payload() + target: Any = payload + for component in path: + target = target[component] + target[extra] = "not allowed" + + with pytest.raises(TruthBenchSchemaError, match="Additional properties"): + validate_run(payload) + + +def test_gate_failures_must_be_non_empty_strings() -> None: + payload = _valid_payload() + payload["gates"][0]["failures"] = [{"message": "not the frozen contract"}] + payload["gates"][0]["failure_count"] = 1 + + with pytest.raises(TruthBenchSchemaError): + validate_run(payload) From 2af2bb928276af435cb6c6575f5783df4c166de4 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 11:40:55 +0530 Subject: [PATCH 08/40] fix: enforce TruthBench result invariants --- benchmarks/truthbench/schema.py | 15 +++++++++++ tests/truthbench/test_schema.py | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/benchmarks/truthbench/schema.py b/benchmarks/truthbench/schema.py index b318080..0738def 100644 --- a/benchmarks/truthbench/schema.py +++ b/benchmarks/truthbench/schema.py @@ -216,6 +216,16 @@ def validate_run(payload: Mapping[str, Any]) -> None: value = record[field] if value is not None and not math.isfinite(value): raise TruthBenchSchemaError(f"record {index} {field} must be a finite number") + if record["sensitive"]: + for field in ("input", "expected_output", "actual_output"): + value = record[field] + if value is not None and not ( + value["value"] is None + and value["display"] == "[REDACTED]" + and bool(value["digest"]) + and value["redacted"] is True + ): + raise TruthBenchSchemaError(f"sensitive record {field} must be redacted") ids = [record["record_id"] for record in records] if len(ids) != len(set(ids)): @@ -229,6 +239,9 @@ def validate_run(payload: Mapping[str, Any]) -> None: raise TruthBenchSchemaError( f"decision record fixture hash is missing for {record['fixture_id']}" ) + record_domains = {record["domain"] for record in records} + if set(fixture_hashes) != record_domains: + raise TruthBenchSchemaError("fixture hash domains do not match record domains") if payload["summary"]["records"] != len(ids): raise TruthBenchSchemaError("record aggregate does not match records") @@ -238,6 +251,8 @@ def validate_run(payload: Mapping[str, Any]) -> None: raise TruthBenchSchemaError( f"gate failure aggregate does not match failures for {gate['name']}" ) + if gate["passed"] is not (gate["failure_count"] == 0): + raise TruthBenchSchemaError(f"gate passed claim is inconsistent for {gate['name']}") passed = all(gate["passed"] for gate in payload["gates"]) if payload["summary"]["overall_passed"] is not passed: diff --git a/tests/truthbench/test_schema.py b/tests/truthbench/test_schema.py index 57b8e37..d59cf03 100644 --- a/tests/truthbench/test_schema.py +++ b/tests/truthbench/test_schema.py @@ -110,6 +110,14 @@ def test_each_record_must_have_a_matching_fixture_hash() -> None: validate_run(payload) +def test_each_declared_fixture_hash_must_have_a_decision_record() -> None: + payload = _valid_payload() + payload["fixture_hashes"]["finance"] = "def456" + + with pytest.raises(TruthBenchSchemaError, match="fixture hash domains"): + validate_run(payload) + + def test_record_aggregate_must_match_record_count() -> None: payload = _valid_payload() payload["summary"]["records"] = 9 @@ -129,6 +137,15 @@ def test_gate_failure_aggregate_must_match_failures() -> None: validate_run(payload) +def test_gate_cannot_pass_when_it_has_failures() -> None: + payload = _valid_payload() + payload["gates"][0]["failure_count"] = 1 + payload["gates"][0]["failures"] = ["record changed"] + + with pytest.raises(TruthBenchSchemaError, match="gate passed claim"): + validate_run(payload) + + def test_overall_passed_claim_cannot_hide_a_failed_gate() -> None: payload = _valid_payload() payload["gates"][0] = GateResult( @@ -171,3 +188,30 @@ def test_gate_failures_must_be_non_empty_strings() -> None: with pytest.raises(TruthBenchSchemaError): validate_run(payload) + + +@pytest.mark.parametrize("field", ["input", "expected_output", "actual_output"]) +def test_sensitive_record_typed_values_must_remain_redacted(field: str) -> None: + payload = _valid_payload() + record = payload["records"][0] + record["sensitive"] = True + redacted = { + **record["input"], + "value": None, + "display": "[REDACTED]", + "digest": "a" * 64, + "redacted": True, + } + record["input"] = deepcopy(redacted) + if field != "input": + record[field] = deepcopy(redacted) + record[f"{field}_type"] = "python.str" + record[field].update( + value="synthetic-placeholder", + display="synthetic-placeholder", + digest=None, + redacted=False, + ) + + with pytest.raises(TruthBenchSchemaError, match=f"sensitive record {field}"): + validate_run(payload) From 18739ceb44d9980408ac8c43005d6520fb96a2c3 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 11:45:57 +0530 Subject: [PATCH 09/40] fix: reject non-finite TruthBench payload values --- benchmarks/truthbench/schema.py | 43 +++++++++++++++++++++++---------- tests/truthbench/test_schema.py | 26 ++++++++++++++++++++ 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/benchmarks/truthbench/schema.py b/benchmarks/truthbench/schema.py index 0738def..d9d8474 100644 --- a/benchmarks/truthbench/schema.py +++ b/benchmarks/truthbench/schema.py @@ -183,7 +183,6 @@ class TruthBenchSchemaError(ValueError): } _VALIDATOR = jsonschema.Draft202012Validator(RESULT_SCHEMA) -_SCORE_FIELDS = ("confidence", "trust_before", "trust_after", "trust_delta") def _schema_path(error: jsonschema.ValidationError) -> str: @@ -205,17 +204,35 @@ def _validate_schema(payload: Mapping[str, Any]) -> None: ) from exc +def _child_path(path: str, component: str | int) -> str: + if isinstance(component, int): + return f"{path}[{component}]" + if component.isidentifier(): + return f"{path}.{component}" + return f"{path}[]" + + +def _validate_finite_numbers(value: Any, path: str = "$") -> None: + if isinstance(value, float) and not math.isfinite(value): + raise TruthBenchSchemaError(f"{path} contains a non-finite number") + if isinstance(value, Mapping): + for key, item in value.items(): + component = key if isinstance(key, str) else "" + _validate_finite_numbers(item, _child_path(path, component)) + elif isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _validate_finite_numbers(item, _child_path(path, index)) + + def validate_run(payload: Mapping[str, Any]) -> None: """Validate a serialized ``RunResult`` and its aggregate integrity.""" - _validate_schema(payload) + copied_payload = dict(payload) + _validate_finite_numbers(copied_payload) + _validate_schema(copied_payload) - records = payload["records"] - for index, record in enumerate(records): - for field in _SCORE_FIELDS: - value = record[field] - if value is not None and not math.isfinite(value): - raise TruthBenchSchemaError(f"record {index} {field} must be a finite number") + records = copied_payload["records"] + for record in records: if record["sensitive"]: for field in ("input", "expected_output", "actual_output"): value = record[field] @@ -231,7 +248,7 @@ def validate_run(payload: Mapping[str, Any]) -> None: if len(ids) != len(set(ids)): raise TruthBenchSchemaError("duplicate decision record id") - fixture_hashes = payload["fixture_hashes"] + fixture_hashes = copied_payload["fixture_hashes"] for record in records: domain = record["domain"] fixture_domain = record["fixture_id"].rsplit(":", 1)[-1] @@ -243,10 +260,10 @@ def validate_run(payload: Mapping[str, Any]) -> None: if set(fixture_hashes) != record_domains: raise TruthBenchSchemaError("fixture hash domains do not match record domains") - if payload["summary"]["records"] != len(ids): + if copied_payload["summary"]["records"] != len(ids): raise TruthBenchSchemaError("record aggregate does not match records") - for gate in payload["gates"]: + for gate in copied_payload["gates"]: if gate["failure_count"] != len(gate["failures"]): raise TruthBenchSchemaError( f"gate failure aggregate does not match failures for {gate['name']}" @@ -254,6 +271,6 @@ def validate_run(payload: Mapping[str, Any]) -> None: if gate["passed"] is not (gate["failure_count"] == 0): raise TruthBenchSchemaError(f"gate passed claim is inconsistent for {gate['name']}") - passed = all(gate["passed"] for gate in payload["gates"]) - if payload["summary"]["overall_passed"] is not passed: + passed = all(gate["passed"] for gate in copied_payload["gates"]) + if copied_payload["summary"]["overall_passed"] is not passed: raise TruthBenchSchemaError("overall gate claim is inconsistent") diff --git a/tests/truthbench/test_schema.py b/tests/truthbench/test_schema.py index d59cf03..7e382e4 100644 --- a/tests/truthbench/test_schema.py +++ b/tests/truthbench/test_schema.py @@ -102,6 +102,32 @@ def test_non_finite_record_scores_are_rejected(field: str, value: float) -> None validate_run(payload) +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_non_finite_typed_value_content_is_rejected_at_its_path(value: float) -> None: + payload = _valid_payload() + payload["records"][0]["input"]["value"] = value + + with pytest.raises( + TruthBenchSchemaError, + match=r"^\$\.records\[0\]\.input\.value contains a non-finite number$", + ): + validate_run(payload) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_non_finite_nested_summary_content_is_rejected_at_its_path( + value: float, +) -> None: + payload = _valid_payload() + payload["summary"]["metrics"] = [{"score": value}] + + with pytest.raises( + TruthBenchSchemaError, + match=r"^\$\.summary\.metrics\[0\]\.score contains a non-finite number$", + ): + validate_run(payload) + + def test_each_record_must_have_a_matching_fixture_hash() -> None: payload = _valid_payload() payload["fixture_hashes"] = {"finance": "def456"} From 45d08478c7cab90c0b400ee5c91a2e3ef73166b3 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:29:56 +0530 Subject: [PATCH 10/40] feat: add complete TruthBench fixture oracle --- benchmarks/truthbench/fixtures/__init__.py | 64 +++++ benchmarks/truthbench/fixtures/base.py | 313 +++++++++++++++++++++ tests/truthbench/conftest.py | 32 +++ tests/truthbench/test_fixtures.py | 82 ++++++ 4 files changed, 491 insertions(+) create mode 100644 benchmarks/truthbench/fixtures/__init__.py create mode 100644 benchmarks/truthbench/fixtures/base.py create mode 100644 tests/truthbench/conftest.py create mode 100644 tests/truthbench/test_fixtures.py diff --git a/benchmarks/truthbench/fixtures/__init__.py b/benchmarks/truthbench/fixtures/__init__.py new file mode 100644 index 0000000..20f5f25 --- /dev/null +++ b/benchmarks/truthbench/fixtures/__init__.py @@ -0,0 +1,64 @@ +"""Fixture registry for the independent TruthBench oracle.""" + +from __future__ import annotations + +from typing import Callable + +import pandas as pd + +from .base import FixtureBuilder, FixtureError, TruthFixture + +_BUILDERS: dict[str, Callable[[int], TruthFixture]] = {} +DOMAINS: tuple[str, ...] = ("minimal",) + + +def register_fixture(domain: str, builder: Callable[[int], TruthFixture]) -> None: + if not isinstance(domain, str) or not domain: + raise ValueError("fixture domain must be a non-empty string") + if domain in _BUILDERS: + raise ValueError(f"fixture domain already registered: {domain}") + _BUILDERS[domain] = builder + + +def _minimal(seed: int) -> TruthFixture: + frame = pd.DataFrame( + { + "name": ["alpha", "beta"], + "amount": [float(seed % 10), float((seed % 10) + 1)], + }, + index=["r1", "r2"], + ) + builder = FixtureBuilder( + "v1", + "minimal", + frame, + seed=seed, + schema={"columns": ["name", "amount"]}, + policy={"locale": "en_US"}, + protected_columns=("name",), + ) + builder.inject("r2", "amount", "2.50", "repair", expected=2.5, family="numeric-format") + return builder.build() + + +register_fixture("minimal", _minimal) + + +def build_fixture(domain: str, seed: int = 1729) -> TruthFixture: + try: + builder = _BUILDERS[domain] + except KeyError as exc: + raise FixtureError(f"unknown fixture domain: {domain!r}") from exc + fixture = builder(int(seed)) + fixture.validate() + return fixture + + +__all__ = [ + "DOMAINS", + "FixtureBuilder", + "FixtureError", + "TruthFixture", + "build_fixture", + "register_fixture", +] diff --git a/benchmarks/truthbench/fixtures/base.py b/benchmarks/truthbench/fixtures/base.py new file mode 100644 index 0000000..3d3a77f --- /dev/null +++ b/benchmarks/truthbench/fixtures/base.py @@ -0,0 +1,313 @@ +"""Independent, deterministic fixture construction primitives.""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +import pandas as pd + +from ..exact import canonical_json, encode_typed +from ..models import UNSET, CaseExpectation, Disposition, GoldCell + + +class FixtureError(ValueError): + """A fixture violates the TruthBench construction contract.""" + + +_HASH_KEY = b"truthbench-fixture-hash-v1" +_SYNTHETIC_EMAIL = re.compile(r"@[A-Za-z0-9.-]+\.invalid(?:$|[^A-Za-z0-9])") +_SYNTHETIC_ID = re.compile(r"(?:^|[^A-Za-z0-9])TB-[A-Za-z0-9-]+(?:$|[^A-Za-z0-9])") +_SYNTHETIC_PHONE = re.compile(r"(?:^|[^0-9])555[- .]?01[0-9]{2}(?:$|[^0-9])") + + +def _synthetic_pii(value: Any) -> bool: + if not isinstance(value, str): + return False + return bool( + _SYNTHETIC_EMAIL.search(value) + or _SYNTHETIC_ID.search(value) + or _SYNTHETIC_PHONE.search(value) + ) + + +def _typed(value: Any, dtype: Any = None, *, sensitive: bool = False) -> dict[str, Any]: + return encode_typed( + value, + dtype=dtype, + sensitive=sensitive, + digest_key=_HASH_KEY if sensitive else None, + ).to_dict() + + +@dataclass(frozen=True) +class TruthFixture: + """A pristine/adversarial pair and complete cell-level oracle.""" + + version: str + domain: str + seed: int + pristine: pd.DataFrame + frame: pd.DataFrame + cells: tuple[GoldCell, ...] + schema: Mapping[str, Any] + policy: Mapping[str, Any] + protected_columns: tuple[str, ...] + pii_canaries: Mapping[str, str] + row_cases: tuple[CaseExpectation, ...] + schema_cases: tuple[CaseExpectation, ...] + fixture_hash: str + + @property + def adversarial(self) -> pd.DataFrame: + return self.frame + + @property + def pristine_frame(self) -> pd.DataFrame: + return self.pristine + + @property + def adversarial_frame(self) -> pd.DataFrame: + return self.frame + + def validate(self) -> None: + if not self.version or not self.domain: + raise FixtureError("fixture version and domain are required") + if not self.frame.index.is_unique or self.frame.index.hasnans: + raise FixtureError("frame row IDs must be unique and non-missing") + keys = [ + (str(row), str(column)) for row in self.frame.index for column in self.frame.columns + ] + labels = [(cell.row_id, cell.column) for cell in self.cells] + if len(labels) != len(set(labels)) or set(labels) != set(keys): + raise FixtureError("fixture must have exactly one label per physical cell") + for cell in self.cells: + if cell.fixture_version != self.version or cell.domain != self.domain: + raise FixtureError("cell identity does not match fixture") + if cell.disposition is Disposition.REPAIR and cell.expected_output is None: + raise FixtureError(f"repair cell {cell.cell_id} has no expected output") + if cell.sensitive and (not cell.canary_id or cell.canary_id not in self.pii_canaries): + raise FixtureError(f"sensitive cell {cell.cell_id} has no canary") + for case in (*self.row_cases, *self.schema_cases): + if case.domain != self.domain or case.fixture_version != self.version: + raise FixtureError("case identity does not match fixture") + if any(case.kind != "row" for case in self.row_cases): + raise FixtureError("row_cases must contain row expectations") + if any(case.kind != "schema" for case in self.schema_cases): + raise FixtureError("schema_cases must contain schema expectations") + if self.fixture_hash: + payload = self.to_dict() + payload.pop("fixture_hash", None) + expected_hash = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + if self.fixture_hash != expected_hash: + raise FixtureError("fixture hash does not match fixture contents") + + def to_dict(self) -> dict[str, Any]: + """Return a privacy-safe, JSON-ready representation.""" + + def frame_payload(frame: pd.DataFrame) -> list[dict[str, Any]]: + by_key = {(c.row_id, c.column): c for c in self.cells} + rows: list[dict[str, Any]] = [] + for row in frame.index: + row_id = str(row) + values = { + str(column): _typed( + frame.at[row, column], + frame[column].dtype, + sensitive=by_key[(row_id, str(column))].sensitive, + ) + for column in frame.columns + } + rows.append({"row_id": row_id, "values": values}) + return rows + + canaries = { + key: _typed(value, sensitive=True) for key, value in sorted(self.pii_canaries.items()) + } + return { + "version": self.version, + "domain": self.domain, + "seed": self.seed, + "pristine": frame_payload(self.pristine), + "frame": frame_payload(self.frame), + "cells": [cell.to_dict() for cell in self.cells], + "schema": dict(self.schema), + "policy": dict(self.policy), + "protected_columns": list(self.protected_columns), + "pii_canaries": canaries, + "row_cases": [case.to_dict() for case in self.row_cases], + "schema_cases": [case.to_dict() for case in self.schema_cases], + "fixture_hash": self.fixture_hash, + } + + +class FixtureBuilder: + """Build a fixture while maintaining a complete, atomic label matrix.""" + + def __init__( + self, + version: str, + domain: str, + frame: pd.DataFrame, + *, + seed: int = 1729, + schema: Mapping[str, Any] | None = None, + policy: Mapping[str, Any] | None = None, + protected_columns: tuple[str, ...] | list[str] = (), + row_cases: tuple[CaseExpectation, ...] | list[CaseExpectation] = (), + schema_cases: tuple[CaseExpectation, ...] | list[CaseExpectation] = (), + ) -> None: + if not isinstance(frame, pd.DataFrame): + raise TypeError("frame must be a pandas DataFrame") + if frame.empty or frame.columns.empty: + raise FixtureError("frame must contain at least one row and column") + if not version or not domain: + raise FixtureError("version and domain are required") + if frame.index.hasnans or not frame.index.is_unique: + raise FixtureError("row IDs must be unique and non-missing") + row_ids = [str(row) for row in frame.index] + if any(not row_id for row_id in row_ids) or len(row_ids) != len(set(row_ids)): + raise FixtureError("row IDs must be unique and non-missing") + if any(not isinstance(column, str) or not column for column in frame.columns): + raise FixtureError("column names must be non-empty strings") + if len(set(frame.columns)) != len(frame.columns): + raise FixtureError("column names must be unique") + + self.version = version + self.domain = domain + self.seed = seed + self.pristine = frame.copy(deep=True) + self.pristine.index = row_ids + self.frame = frame.copy(deep=True) + self.frame.index = row_ids + self.schema = dict( + schema + or { + "columns": list(frame.columns), + "dtypes": {str(c): str(frame[c].dtype) for c in frame.columns}, + } + ) + self.policy = dict(policy or {}) + self.protected_columns = tuple(protected_columns) + unknown_protected = set(self.protected_columns) - set(frame.columns) + if unknown_protected: + raise FixtureError(f"unknown protected columns: {sorted(unknown_protected)}") + self._labels = { + (row_id, str(column)): GoldCell.create( + version, domain, row_id, str(column), Disposition.PRESERVE, family="background" + ) + for row_id in row_ids + for column in frame.columns + } + self._canaries: dict[str, str] = {} + self._row_cases = list(row_cases) + self._schema_cases = list(schema_cases) + + def inject( + self, + row_id: str, + column: str, + value: Any, + disposition: Disposition | str, + *, + expected: Any = UNSET, + expected_dtype: Any = None, + family: str, + sensitive: bool = False, + ) -> None: + key = (str(row_id), str(column)) + if key not in self._labels: + raise FixtureError(f"unknown cell {key}") + if not isinstance(family, str) or not family: + raise FixtureError("family is required") + try: + parsed = Disposition(disposition) + except ValueError as exc: + raise FixtureError(f"unknown disposition: {disposition!r}") from exc + if parsed is Disposition.REPAIR and expected is UNSET: + raise FixtureError("repair cells require an expected output") + if sensitive and not _synthetic_pii(value): + raise FixtureError("sensitive values must use synthetic PII domains") + + canary_id = None + if sensitive: + canary_id = f"{self.domain}-{key[0]}-{key[1]}" + if expected_dtype is None and expected is not UNSET: + expected_dtype = self.pristine[key[1]].dtype + candidate = GoldCell.create( + self.version, + self.domain, + key[0], + key[1], + parsed, + expected_output=expected, + expected_dtype=expected_dtype, + family=family, + sensitive=sensitive, + canary_id=canary_id, + ) + previous = self._labels[key] + if ( + previous.disposition is Disposition.REPAIR + and candidate.disposition is Disposition.REPAIR + and previous.expected_output != candidate.expected_output + ): + raise FixtureError(f"contradictory repair output for cell {key}") + + # All validation above happens before either mutable structure changes. + if previous.canary_id and previous.canary_id != canary_id: + self._canaries.pop(previous.canary_id, None) + if canary_id: + self._canaries[canary_id] = value + # A corruption may intentionally change a column's scalar type. Cast + # to object first so pandas does not coerce or emit a FutureWarning. + self.frame[key[1]] = self.frame[key[1]].astype(object) + self.frame.at[key[0], key[1]] = value + self._labels[key] = candidate + + def add_row_case(self, name: str, disposition: Disposition | str, **kwargs: Any) -> None: + self._row_cases.append( + CaseExpectation.create(self.version, self.domain, "row", name, disposition, **kwargs) + ) + + def add_schema_case(self, name: str, disposition: Disposition | str, **kwargs: Any) -> None: + self._schema_cases.append( + CaseExpectation.create( + self.version, self.domain, "schema", name, disposition, **kwargs + ) + ) + + def build(self) -> TruthFixture: + cells = tuple( + self._labels[(str(row), str(column))] + for row in self.frame.index + for column in self.frame.columns + ) + fixture = TruthFixture( + version=self.version, + domain=self.domain, + seed=self.seed, + pristine=self.pristine.copy(deep=True), + frame=self.frame.copy(deep=True), + cells=cells, + schema=dict(self.schema), + policy=dict(self.policy), + protected_columns=self.protected_columns, + pii_canaries=dict(self._canaries), + row_cases=tuple(self._row_cases), + schema_cases=tuple(self._schema_cases), + fixture_hash="", + ) + fixture.validate() + payload = fixture.to_dict() + payload.pop("fixture_hash", None) + digest = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + result = TruthFixture(**{**fixture.__dict__, "fixture_hash": digest}) + result.validate() + return result + + +BuilderFactory = Callable[[int], TruthFixture] diff --git a/tests/truthbench/conftest.py b/tests/truthbench/conftest.py new file mode 100644 index 0000000..8187e28 --- /dev/null +++ b/tests/truthbench/conftest.py @@ -0,0 +1,32 @@ +"""Focused fixtures for the initial TruthBench fixture infrastructure tests.""" + +from __future__ import annotations + +import pandas as pd +import pytest +from benchmarks.truthbench.fixtures.base import FixtureBuilder + + +@pytest.fixture +def minimal_fixture(): + frame = pd.DataFrame( + {"name": ["alpha", "beta"], "amount": [1.0, 2.0]}, + index=["r1", "r2"], + ) + builder = FixtureBuilder( + "v1", + "minimal", + frame, + schema={"columns": ["name", "amount"]}, + policy={"locale": "en_US"}, + protected_columns=("name",), + ) + builder.inject( + "r2", + "amount", + "2.50", + "repair", + expected=2.5, + family="numeric-format", + ) + return builder.build() diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py new file mode 100644 index 0000000..a01b34e --- /dev/null +++ b/tests/truthbench/test_fixtures.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json + +import pandas as pd +import pytest +from benchmarks.truthbench import Disposition +from benchmarks.truthbench.fixtures import build_fixture +from benchmarks.truthbench.fixtures.base import FixtureBuilder, FixtureError + +DOMAINS = ("minimal",) + + +@pytest.mark.parametrize("domain", DOMAINS) +def test_every_physical_cell_has_exactly_one_label(domain: str) -> None: + fixture = build_fixture(domain, seed=1729) + expected = {(str(row), str(col)) for row in fixture.frame.index for col in fixture.frame} + actual = {(cell.row_id, cell.column) for cell in fixture.cells} + assert actual == expected + assert len(fixture.cells) == len(expected) + fixture.validate() + + +def test_injected_case_replaces_default_preserve_label(minimal_fixture) -> None: + cell = next( + cell for cell in minimal_fixture.cells if cell.row_id == "r2" and cell.column == "amount" + ) + assert cell.disposition is Disposition.REPAIR + assert cell.expected_output is not None + assert cell.expected_output.type_label == "python.float[float64]" + assert minimal_fixture.frame.at["r2", "amount"] == "2.50" + + +def test_sensitive_cells_have_canary_ids_and_no_raw_canary_in_hash(minimal_fixture) -> None: + frame = pd.DataFrame( + {"notes": ["ordinary", "tb.person+7@example.invalid"]}, index=["r1", "r2"] + ) + builder = FixtureBuilder("v1", "minimal", frame) + builder.inject( + "r2", + "notes", + "tb.person+7@example.invalid", + "flag", + family="email", + sensitive=True, + ) + fixture = builder.build() + sensitive = fixture.cells[1] + assert sensitive.sensitive and sensitive.canary_id + assert sensitive.canary_id in fixture.pii_canaries + assert "tb.person+7@example.invalid" not in json.dumps(fixture.to_dict()) + + +def test_builder_rejects_missing_rows_duplicate_rows_unknown_cells_and_bad_repairs() -> None: + with pytest.raises(FixtureError, match="row IDs"): + FixtureBuilder("v1", "minimal", pd.DataFrame({"a": [1, 2]}, index=["r", "r"])) + + builder = FixtureBuilder("v1", "minimal", pd.DataFrame({"a": [1]}, index=["r1"])) + with pytest.raises(FixtureError, match="unknown cell"): + builder.inject("r9", "a", 2, "flag", family="x") + with pytest.raises(FixtureError, match="unknown cell"): + builder.inject("r1", "missing", 2, "flag", family="x") + with pytest.raises(FixtureError, match="expected"): + builder.inject("r1", "a", "2", "repair", family="x") + + +def test_repair_expected_output_cannot_be_contradictory() -> None: + builder = FixtureBuilder("v1", "minimal", pd.DataFrame({"a": [1]}, index=["r1"])) + builder.inject("r1", "a", "2", "repair", expected=2, family="x") + with pytest.raises(FixtureError, match="contradictory"): + builder.inject("r1", "a", "3", "repair", expected=3, family="x") + assert builder.frame.at["r1", "a"] == "2" + + +def test_fixture_hash_is_stable_and_includes_metadata() -> None: + first = build_fixture("minimal", seed=1729) + second = build_fixture("minimal", seed=1729) + assert first.fixture_hash == second.fixture_hash + altered = FixtureBuilder( + "v1", "minimal", first.pristine.copy(), schema={"different": True} + ).build() + assert altered.fixture_hash != first.fixture_hash From b27aaf8c8d32004d85c3fe1398185701f21f5cec Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:30:40 +0530 Subject: [PATCH 11/40] fix: allow synthetic numeric PII canaries --- benchmarks/truthbench/fixtures/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/truthbench/fixtures/base.py b/benchmarks/truthbench/fixtures/base.py index 3d3a77f..c74d445 100644 --- a/benchmarks/truthbench/fixtures/base.py +++ b/benchmarks/truthbench/fixtures/base.py @@ -25,6 +25,8 @@ class FixtureError(ValueError): def _synthetic_pii(value: Any) -> bool: + if isinstance(value, int) and not isinstance(value, bool): + value = str(value) if not isinstance(value, str): return False return bool( @@ -56,7 +58,7 @@ class TruthFixture: schema: Mapping[str, Any] policy: Mapping[str, Any] protected_columns: tuple[str, ...] - pii_canaries: Mapping[str, str] + pii_canaries: Mapping[str, Any] row_cases: tuple[CaseExpectation, ...] schema_cases: tuple[CaseExpectation, ...] fixture_hash: str @@ -202,7 +204,7 @@ def __init__( for row_id in row_ids for column in frame.columns } - self._canaries: dict[str, str] = {} + self._canaries: dict[str, Any] = {} self._row_cases = list(row_cases) self._schema_cases = list(schema_cases) From 8c9b1e6d079abf7e1265c10e8b352f60e391c5fa Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:31:27 +0530 Subject: [PATCH 12/40] fix: validate fixture identity and canary inputs --- benchmarks/truthbench/fixtures/base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/truthbench/fixtures/base.py b/benchmarks/truthbench/fixtures/base.py index c74d445..8e6f951 100644 --- a/benchmarks/truthbench/fixtures/base.py +++ b/benchmarks/truthbench/fixtures/base.py @@ -86,9 +86,15 @@ def validate(self) -> None: labels = [(cell.row_id, cell.column) for cell in self.cells] if len(labels) != len(set(labels)) or set(labels) != set(keys): raise FixtureError("fixture must have exactly one label per physical cell") + cell_ids = [cell.cell_id for cell in self.cells] + if len(cell_ids) != len(set(cell_ids)): + raise FixtureError("fixture contains duplicate cell IDs") for cell in self.cells: if cell.fixture_version != self.version or cell.domain != self.domain: raise FixtureError("cell identity does not match fixture") + expected_id = f"{self.version}:{self.domain}:{cell.row_id}:{cell.column}" + if cell.cell_id != expected_id: + raise FixtureError(f"cell {cell.cell_id} has an invalid stable ID") if cell.disposition is Disposition.REPAIR and cell.expected_output is None: raise FixtureError(f"repair cell {cell.cell_id} has no expected output") if cell.sensitive and (not cell.canary_id or cell.canary_id not in self.pii_canaries): @@ -159,6 +165,7 @@ def __init__( schema: Mapping[str, Any] | None = None, policy: Mapping[str, Any] | None = None, protected_columns: tuple[str, ...] | list[str] = (), + pii_canaries: Mapping[str, Any] | None = None, row_cases: tuple[CaseExpectation, ...] | list[CaseExpectation] = (), schema_cases: tuple[CaseExpectation, ...] | list[CaseExpectation] = (), ) -> None: @@ -204,7 +211,9 @@ def __init__( for row_id in row_ids for column in frame.columns } - self._canaries: dict[str, Any] = {} + self._canaries: dict[str, Any] = dict(pii_canaries or {}) + if any(not _synthetic_pii(value) for value in self._canaries.values()): + raise FixtureError("PII canaries must use synthetic domains") self._row_cases = list(row_cases) self._schema_cases = list(schema_cases) From 05cb5162d0f4bc8ff062753502292e4434b8644c Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:32:10 +0530 Subject: [PATCH 13/40] fix: preserve explicit repair output typing --- benchmarks/truthbench/fixtures/base.py | 2 -- tests/truthbench/test_fixtures.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/truthbench/fixtures/base.py b/benchmarks/truthbench/fixtures/base.py index 8e6f951..6b655ed 100644 --- a/benchmarks/truthbench/fixtures/base.py +++ b/benchmarks/truthbench/fixtures/base.py @@ -246,8 +246,6 @@ def inject( canary_id = None if sensitive: canary_id = f"{self.domain}-{key[0]}-{key[1]}" - if expected_dtype is None and expected is not UNSET: - expected_dtype = self.pristine[key[1]].dtype candidate = GoldCell.create( self.version, self.domain, diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py index a01b34e..df795c8 100644 --- a/tests/truthbench/test_fixtures.py +++ b/tests/truthbench/test_fixtures.py @@ -27,7 +27,7 @@ def test_injected_case_replaces_default_preserve_label(minimal_fixture) -> None: ) assert cell.disposition is Disposition.REPAIR assert cell.expected_output is not None - assert cell.expected_output.type_label == "python.float[float64]" + assert cell.expected_output.type_label == "python.float" assert minimal_fixture.frame.at["r2", "amount"] == "2.50" From 3499ab12529a209238d7c0d26165052a7010d525 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:38:09 +0530 Subject: [PATCH 14/40] fix: require whole-value synthetic PII canaries --- benchmarks/truthbench/fixtures/base.py | 15 +++++++-------- tests/truthbench/test_fixtures.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/benchmarks/truthbench/fixtures/base.py b/benchmarks/truthbench/fixtures/base.py index 6b655ed..758a902 100644 --- a/benchmarks/truthbench/fixtures/base.py +++ b/benchmarks/truthbench/fixtures/base.py @@ -6,6 +6,7 @@ import re from collections.abc import Callable, Mapping from dataclasses import dataclass +from numbers import Integral from typing import Any import pandas as pd @@ -19,20 +20,18 @@ class FixtureError(ValueError): _HASH_KEY = b"truthbench-fixture-hash-v1" -_SYNTHETIC_EMAIL = re.compile(r"@[A-Za-z0-9.-]+\.invalid(?:$|[^A-Za-z0-9])") -_SYNTHETIC_ID = re.compile(r"(?:^|[^A-Za-z0-9])TB-[A-Za-z0-9-]+(?:$|[^A-Za-z0-9])") -_SYNTHETIC_PHONE = re.compile(r"(?:^|[^0-9])555[- .]?01[0-9]{2}(?:$|[^0-9])") +_SYNTHETIC_EMAIL = re.compile(r"[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+\.invalid") +_SYNTHETIC_ID = re.compile(r"TB-[A-Za-z0-9-]+") +_SYNTHETIC_PHONE = re.compile(r"555[- .]?01[0-9]{2}") def _synthetic_pii(value: Any) -> bool: - if isinstance(value, int) and not isinstance(value, bool): + if isinstance(value, Integral) and not isinstance(value, bool): value = str(value) if not isinstance(value, str): return False - return bool( - _SYNTHETIC_EMAIL.search(value) - or _SYNTHETIC_ID.search(value) - or _SYNTHETIC_PHONE.search(value) + return any( + pattern.fullmatch(value) for pattern in (_SYNTHETIC_EMAIL, _SYNTHETIC_ID, _SYNTHETIC_PHONE) ) diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py index df795c8..4fe90f5 100644 --- a/tests/truthbench/test_fixtures.py +++ b/tests/truthbench/test_fixtures.py @@ -51,6 +51,24 @@ def test_sensitive_cells_have_canary_ids_and_no_raw_canary_in_hash(minimal_fixtu assert "tb.person+7@example.invalid" not in json.dumps(fixture.to_dict()) +@pytest.mark.parametrize( + "value", + [ + "alice@gmail.com TB-1", + "TB-1 alice@gmail.com", + "415-555-0107", + "555-0107 9876543210", + "alice@gmail.com", + "123-45-6789", + ], +) +def test_sensitive_canaries_must_be_whole_value_synthetic_forms(value: str) -> None: + frame = pd.DataFrame({"notes": ["ordinary"]}, index=["r1"]) + builder = FixtureBuilder("v1", "minimal", frame) + with pytest.raises(FixtureError, match="synthetic PII"): + builder.inject("r1", "notes", value, "flag", family="pii", sensitive=True) + + def test_builder_rejects_missing_rows_duplicate_rows_unknown_cells_and_bad_repairs() -> None: with pytest.raises(FixtureError, match="row IDs"): FixtureBuilder("v1", "minimal", pd.DataFrame({"a": [1, 2]}, index=["r", "r"])) From d3caf50af1515a0aeb91f67934e5e807db1d3e94 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:48:22 +0530 Subject: [PATCH 15/40] feat: add first TruthBench domain corpus --- .superpowers/sdd/task-4-report.md | 63 +++++ benchmarks/truthbench/fixtures/__init__.py | 7 +- benchmarks/truthbench/fixtures/crm.py | 257 +++++++++++++++++++ benchmarks/truthbench/fixtures/finance.py | 204 +++++++++++++++ benchmarks/truthbench/fixtures/healthcare.py | 208 +++++++++++++++ benchmarks/truthbench/fixtures/retail.py | 194 ++++++++++++++ tests/truthbench/test_fixtures.py | 40 ++- 7 files changed, 971 insertions(+), 2 deletions(-) create mode 100644 .superpowers/sdd/task-4-report.md create mode 100644 benchmarks/truthbench/fixtures/crm.py create mode 100644 benchmarks/truthbench/fixtures/finance.py create mode 100644 benchmarks/truthbench/fixtures/healthcare.py create mode 100644 benchmarks/truthbench/fixtures/retail.py diff --git a/.superpowers/sdd/task-4-report.md b/.superpowers/sdd/task-4-report.md new file mode 100644 index 0000000..5f21935 --- /dev/null +++ b/.superpowers/sdd/task-4-report.md @@ -0,0 +1,63 @@ +# Task 4 Report: Add finance, healthcare, retail, and CRM gold datasets + +## Status + +DONE. The four deterministic 16-row domain builders are registered and covered by focused and adjacent TruthBench tests. + +## TDD evidence + +### RED + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Observed 16 failures before implementation: each new domain was absent from the fixture registry (`FixtureError: unknown fixture domain`). + +### GREEN + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result: `28 passed`. + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py tests/truthbench/test_models_exact.py tests/truthbench/test_schema.py -q --no-cov +``` + +Result: all focused, model, and schema tests green. + +Static checks: + +```text +python -m ruff check benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py +python -m ruff format --check benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py +mypy --ignore-missing-imports --explicit-package-bases benchmarks/truthbench/fixtures tests/truthbench/conftest.py tests/truthbench/test_fixtures.py +git diff --check +``` + +All passed. + +## Files + +- `benchmarks/truthbench/fixtures/finance.py` — finance frame and oracle labels for Apple semantic traps, numeric/currency/date defects, protected ticker conflict, and synthetic PII canaries. +- `benchmarks/truthbench/fixtures/healthcare.py` — healthcare frame and oracle labels for ICD/LOINC, temperature and dose units, FHIR/partial/impossible dates, Unicode, and PHI canaries. +- `benchmarks/truthbench/fixtures/retail.py` — retail frame and oracle labels for leading-zero identifiers, free/return quantities, locale currency formats, mojibake/HTML, multilingual values, and review PII. +- `benchmarks/truthbench/fixtures/crm.py` — CRM frame and oracle labels for Unicode/contact ambiguity, lifecycle contradiction, zero-width/hidden PII, Apple semantic traps, and protected IDs. +- `benchmarks/truthbench/fixtures/__init__.py` — registry entries and stable domain order. +- `tests/truthbench/test_fixtures.py` — domain completeness, disposition, deterministic-seed, and adversarial marker tests. + +## Self-review + +- Every builder emits exactly 16 rows with stable string indexes, fixed `2026-01-15` UTC reference metadata, explicit locale metadata, complete physical-cell labels, and at least 12 injected adversarial cells. +- All four dispositions are represented per domain. Row cases cover exact duplicates and removed rows; schema cases cover added, removed, renamed, reordered, and type-drifted columns without mislabeling absent cells. +- Sensitive values are whole-value synthetic `.invalid`, `TB-*`, or `555-01xx` forms, preserving the base builder's redaction and canary invariants. Zero-width examples are intentionally non-sensitive representation traps. +- Seed values are included only in a deterministic batch marker; same-seed builds produce byte-stable serialized oracle payloads and identical fixture hashes for seeds `1729` and `2718`. +- No FreshData runtime, LLM/provider, network, or external data dependency is used. + +## Concerns + +- The legacy `minimal` fixture remains registered for backwards compatibility; the four Task 4 domains are appended in stable order. A later registry task may choose to retire `minimal` once its callers migrate. +- Row/schema expectations are metadata cases (the physical frame remains rectangular), consistent with the oracle contract that removed rows/columns cannot have cell labels. + diff --git a/benchmarks/truthbench/fixtures/__init__.py b/benchmarks/truthbench/fixtures/__init__.py index 20f5f25..704c334 100644 --- a/benchmarks/truthbench/fixtures/__init__.py +++ b/benchmarks/truthbench/fixtures/__init__.py @@ -6,10 +6,11 @@ import pandas as pd +from . import crm, finance, healthcare, retail from .base import FixtureBuilder, FixtureError, TruthFixture _BUILDERS: dict[str, Callable[[int], TruthFixture]] = {} -DOMAINS: tuple[str, ...] = ("minimal",) +DOMAINS: tuple[str, ...] = ("minimal", "finance", "healthcare", "retail", "crm") def register_fixture(domain: str, builder: Callable[[int], TruthFixture]) -> None: @@ -42,6 +43,10 @@ def _minimal(seed: int) -> TruthFixture: register_fixture("minimal", _minimal) +register_fixture("finance", finance.build) +register_fixture("healthcare", healthcare.build) +register_fixture("retail", retail.build) +register_fixture("crm", crm.build) def build_fixture(domain: str, seed: int = 1729) -> TruthFixture: diff --git a/benchmarks/truthbench/fixtures/crm.py b/benchmarks/truthbench/fixtures/crm.py new file mode 100644 index 0000000..6878d17 --- /dev/null +++ b/benchmarks/truthbench/fixtures/crm.py @@ -0,0 +1,257 @@ +"""Deterministic CRM gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"crm-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "customer_id": [f"TB-CRM-{i:04d}" for i in range(1, 17)], + "first_name": [ + "Ana", + "Jose\u0301", + "Miyuki", + "Zoë", + "Liam", + "Nia", + "Omar", + "Riya", + "Noel", + "Saanvi", + "Evan", + "Ivy", + "Kai", + "Mara", + "Theo", + "Uma", + ], + "last_name": [ + "Stone", + "García", + "Chen", + "Müller", + "Jones", + "Patel", + "Khan", + "Roy", + "Smith", + "Das", + "Lee", + "Brown", + "Cruz", + "Mori", + "Cole", + "Singh", + ], + "email": [ + "ana@example.invalid", + "jose@example.invalid", + "miyuki@example.invalid", + "zoe@example.invalid", + "liam@example.invalid", + "nia@example.invalid", + "omar@example.invalid", + "riya@example.invalid", + "noel@example.invalid", + "saanvi@example.invalid", + "evan@example.invalid", + "ivy@example.invalid", + "kai@example.invalid", + "mara@example.invalid", + "theo@example.invalid", + "uma@example.invalid", + ], + "phone": ["555-0101"] * 16, + "country": [ + "US", + "DE", + "IN", + "GB", + "US", + "CA", + "AU", + "JP", + "BR", + "FR", + "US", + "IN", + "MX", + "ES", + "ZA", + "SG", + ], + "language": [ + "en", + "de", + "hi", + "en", + "en", + "fr", + "en", + "ja", + "pt", + "fr", + "en", + "hi", + "es", + "es", + "en", + "en", + ], + "signup_date": ["2026-01-15"] * 16, + "lifecycle": [ + "lead", + "prospect", + "customer", + "churned", + "lead", + "prospect", + "customer", + "lead", + "prospect", + "customer", + "lead", + "prospect", + "customer", + "lead", + "prospect", + "customer", + ], + "lead_source": [ + "web", + "referral", + "apple", + "partner", + "web", + "event", + "apple", + "search", + "web", + "partner", + "event", + "web", + "referral", + "search", + "web", + "event", + ], + "employer": [ + "Acme", + "Globex", + "Apple", + "Northstar", + "Acme", + "Cedar", + "Apple", + "Delta", + "Evergreen", + "Futura", + "Globex", + "Helios", + "Apple", + "Ion", + "Juniper", + "Kappa", + ], + "notes": ["ordinary"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "crm", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "supported_countries": ["US", "DE", "IN", "GB"], + "supported_languages": ["en", "de", "hi", "ja"], + "protected_customer_id_policy": "preserve", + }, + protected_columns=("customer_id",), + ) + builder.inject( + "crm-02", + "first_name", + "Jose\u0301", + Disposition.REPAIR, + expected="José", + family="combining-unicode-name", + ) + builder.inject( + "crm-03", + "email", + "miyuki@example.invalid", + Disposition.PRESERVE, + family="reserved-invalid-email", + sensitive=True, + ) + builder.inject( + "crm-04", + "phone", + "555 0101", + Disposition.REPAIR, + expected="555-0101", + family="spaced-phone", + ) + builder.inject("crm-05", "country", "US/CA", Disposition.REVIEW, family="ambiguous-country") + builder.inject("crm-06", "language", "en/fr", Disposition.REVIEW, family="ambiguous-language") + builder.inject( + "crm-07", "signup_date", "01/02/2025", Disposition.REVIEW, family="ambiguous-date" + ) + builder.inject( + "crm-08", "lifecycle", "lead|churned", Disposition.REVIEW, family="lifecycle-contradiction" + ) + builder.inject( + "crm-09", + "email", + "TB-CRM-ZW-EMAIL", + Disposition.FLAG, + family="zero-width-email", + sensitive=True, + ) + builder.inject( + "crm-11", + "email", + "lead\u200b@example.invalid", + Disposition.FLAG, + family="zero-width-email", + ) + builder.inject( + "crm-10", "notes", "TB-CRM-SSN-0001", Disposition.FLAG, family="hidden-ssn", sensitive=True + ) + builder.inject( + "crm-03", "lead_source", "apple", Disposition.PRESERVE, family="apple-lead-source" + ) + builder.inject("crm-03", "employer", "Apple", Disposition.PRESERVE, family="apple-employer") + builder.inject( + "crm-16", + "customer_id", + "TB-CRM-CUSTOMER-TAIL", + Disposition.REVIEW, + family="protected-customer-id", + sensitive=True, + ) + builder.add_row_case( + "exact-duplicate-crm-02-crm-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-crm-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case("type-drifted-phone", Disposition.REVIEW, family="type-drifted-column") + return builder.build() diff --git a/benchmarks/truthbench/fixtures/finance.py b/benchmarks/truthbench/fixtures/finance.py new file mode 100644 index 0000000..598f5d0 --- /dev/null +++ b/benchmarks/truthbench/fixtures/finance.py @@ -0,0 +1,204 @@ +"""Deterministic finance gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"fin-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "account_id": [f"TB-FIN-{i:04d}" for i in range(1, 17)], + "asset": [ + "bond", + "fund", + "apple", + "stock", + "cash", + "fx", + "etf", + "bond", + "fund", + "stock", + "cash", + "fx", + "etf", + "bond", + "fund", + "stock", + ], + "company": [ + "Acme Bank", + "Northstar", + "Acme", + "Apple", + "Cedar", + "Delta", + "Evergreen", + "Futura", + "Globex", + "Helios", + "Ion", + "Juniper", + "Kappa", + "Lumen", + "Mosaic", + "Nadir", + ], + "ticker": [ + "ACME", + "NSTAR", + "APPL", + "AAPL", + "CEDR", + "DLTA", + "EVER", + "FUTR", + "GLOB", + "HELI", + "ION", + "JUN", + "KAPP", + "LUME", + "MOSC", + "NADI", + ], + "price": [ + 10.25, + 0.00, + "apple", + 155.20, + -4.5, + 9999999999.99, + "₹1,23,456.70", + 123.45, + 9.99, + 42.0, + 18.75, + 21.0, + 33.0, + 7.5, + 8.25, + 12.0, + ], + "currency": [ + "USD", + "USD", + "USD", + "USD", + "EUR", + "USD", + "INR", + "EUR", + "USD", + "USD", + "USD", + "EUR", + "INR", + "USD", + "USD", + "USD", + ], + "trade_date": ["2026-01-15"] * 16, + "memo": ["ordinary"] * 16, + "balance": [ + 100.0, + 200.0, + 300.0, + 400.0, + 500.0, + 600.0, + 700.0, + 800.0, + 900.0, + 1000.0, + 1100.0, + 1200.0, + 1300.0, + 1400.0, + 1500.0, + 1600.0, + ], + "locale": ["en_US"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "finance", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "currency": "USD", + "supported_currencies": ["USD", "EUR", "INR"], + "protected_ticker_policy": "preserve", + }, + protected_columns=("account_id", "ticker"), + ) + builder.inject("fin-03", "price", "apple", Disposition.REVIEW, family="semantic-apple-price") + builder.inject("fin-04", "company", "Apple", Disposition.PRESERVE, family="apple-company") + builder.inject( + "fin-04", "ticker", "AAPL", Disposition.REVIEW, family="protected-ticker-policy-conflict" + ) + builder.inject("fin-02", "price", "0.00", Disposition.PRESERVE, family="zero-price") + builder.inject("fin-05", "price", -4.5, Disposition.FLAG, family="negative-value") + builder.inject("fin-06", "price", 9999999999.99, Disposition.FLAG, family="extreme-value") + builder.inject( + "fin-07", + "price", + "₹1,23,456.70", + Disposition.REPAIR, + expected=123456.70, + expected_dtype="float64", + family="indian-grouped-currency", + ) + builder.inject("fin-08", "currency", "EUR", Disposition.REVIEW, family="usd-eur-inr-conflict") + builder.inject("fin-09", "currency", "INR", Disposition.REVIEW, family="usd-eur-inr-conflict") + builder.inject( + "fin-10", "trade_date", "01/02/2025", Disposition.REVIEW, family="ambiguous-date-format" + ) + builder.inject( + "fin-11", + "memo", + "tb.finance+memo@example.invalid", + Disposition.FLAG, + family="invisible-pii-memo", + sensitive=True, + ) + builder.inject( + "fin-12", + "memo", + "tb.finance+hidden@example.invalid\u200b", + Disposition.FLAG, + family="invisible-pii-memo", + ) + builder.inject( + "fin-16", + "account_id", + "TB-FIN-ACCOUNT-TAIL", + Disposition.FLAG, + family="tail-row-account-canary", + sensitive=True, + ) + builder.add_row_case( + "exact-duplicate-fin-02-fin-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-fin-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case("type-drifted-price", Disposition.REVIEW, family="type-drifted-column") + return builder.build() diff --git a/benchmarks/truthbench/fixtures/healthcare.py b/benchmarks/truthbench/fixtures/healthcare.py new file mode 100644 index 0000000..2c898b8 --- /dev/null +++ b/benchmarks/truthbench/fixtures/healthcare.py @@ -0,0 +1,208 @@ +"""Deterministic healthcare gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"hc-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "mrn": [f"TB-HC-{i:04d}" for i in range(1, 17)], + "patient_name": [ + "Amina Khan", + "Luca Rossi", + "Marta Silva", + "Noah Reed", + "Iris Chen", + "Sofia Patel", + "Eli Jones", + "Ravi Singh", + "Nora Stone", + "Mina Park", + "Owen Hall", + "Zoe King", + "Leo Cruz", + "Aya Mori", + "Finn Cole", + "Uma Roy", + ], + "diagnosis_code": [ + "R69", + "E11.9", + "I10", + "G rare", + "J45.9", + "M54.5", + "N39.0", + "K21.9", + "L20.9", + "D50.9", + "G43.9", + "H25.1", + "C rare", + "B20", + "A09", + "Z00.0", + ], + "loinc": [ + "LP21258-6", + "2345-7", + "718-7", + "rare-LOINC-123", + "4548-4", + "8310-5", + "2951-2", + "2160-0", + "6690-2", + "26499-4", + "3094-0", + "33747-0", + "rare-LOINC-456", + "1751-7", + "2823-3", + "1975-2", + ], + "temperature": [ + 36.8, + 37.0, + 36.5, + 98.6, + 37.1, + 36.9, + 37.2, + 36.7, + 36.6, + 37.0, + 36.8, + 36.9, + 37.1, + 36.8, + 36.7, + 36.9, + ], + "dose": [ + "5 mg", + "10 mg", + "2.5 mg", + "5000 mcg", + "1 mg", + "20 mg", + "5 mg", + "2 mg", + "8 mg", + "4 mg", + "3 mg", + "6 mg", + "7 mg", + "9 mg", + "1 mg", + "2 mg", + ], + "event_date": ["2026-01-15"] * 16, + "dob": ["1980-01-01"] * 16, + "notes": ["routine"] * 16, + "phone": ["555-0101"] * 16, + "language": ["en"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "healthcare", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "standards": ["FHIR", "ICD-10", "LOINC"], + "temperature_unit": "C", + "protected_dob_policy": "preserve", + }, + protected_columns=("mrn", "dob"), + ) + builder.inject( + "hc-04", "diagnosis_code", "G rare", Disposition.PRESERVE, family="rare-icd-valid" + ) + builder.inject( + "hc-04", "loinc", "rare-LOINC-123", Disposition.PRESERVE, family="rare-loinc-valid" + ) + builder.inject( + "hc-04", "temperature", 98.6, Disposition.REVIEW, family="celsius-fahrenheit-conflict" + ) + builder.inject("hc-01", "dose", "5 mg", Disposition.PRESERVE, family="dose-unit-valid") + builder.inject( + "hc-04", + "dose", + "5000 mcg", + Disposition.REPAIR, + expected=5.0, + expected_dtype="float64", + family="mg-mcg-unit-conversion", + ) + builder.inject("hc-05", "event_date", "2025-01", Disposition.REVIEW, family="partial-date") + builder.inject( + "hc-06", + "event_date", + "2025-01-15T12:00:00Z", + Disposition.REPAIR, + expected="2025-01-15", + family="fhir-date", + ) + builder.inject("hc-07", "event_date", "2025-02-30", Disposition.FLAG, family="impossible-date") + builder.inject( + "hc-08", + "patient_name", + "Jose\u0301", + Disposition.REPAIR, + expected="José", + family="decomposed-unicode", + ) + builder.inject( + "hc-09", "notes", "TB-HC-PHI-0001", Disposition.FLAG, family="phi-in-notes", sensitive=True + ) + builder.inject( + "hc-10", + "phone", + "555-0107", + Disposition.PRESERVE, + family="synthetic-phone", + sensitive=True, + ) + builder.inject( + "hc-11", "notes", "TB-HC-MRN-NOTE", Disposition.FLAG, family="mrn-in-notes", sensitive=True + ) + builder.inject( + "hc-12", "notes", "555-0112", Disposition.FLAG, family="phone-in-notes", sensitive=True + ) + builder.inject( + "hc-16", + "mrn", + "TB-HC-MRN-TAIL", + Disposition.REVIEW, + family="mrn-tail-canary", + sensitive=True, + ) + builder.inject( + "hc-15", "dob", "01/01/1980", Disposition.REVIEW, family="protected-dob-repair-conflict" + ) + builder.add_row_case("exact-duplicate-hc-02-hc-03", Disposition.FLAG, family="exact-duplicate") + builder.add_row_case("removed-hc-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case( + "type-drifted-temperature", Disposition.REVIEW, family="type-drifted-column" + ) + return builder.build() diff --git a/benchmarks/truthbench/fixtures/retail.py b/benchmarks/truthbench/fixtures/retail.py new file mode 100644 index 0000000..be39bbf --- /dev/null +++ b/benchmarks/truthbench/fixtures/retail.py @@ -0,0 +1,194 @@ +"""Deterministic retail gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"ret-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "sku": [ + "001234", + "000007", + "123456", + "888888", + "100001", + "100002", + "100003", + "100004", + "100005", + "100006", + "100007", + "100008", + "100009", + "100010", + "100011", + "100012", + ], + "gtin": [ + "00012345678905", + "00000012345678", + "12345678901234", + "88888888888888", + "10000000000001", + "10000000000002", + "10000000000003", + "10000000000004", + "10000000000005", + "10000000000006", + "10000000000007", + "10000000000008", + "10000000000009", + "10000000000010", + "10000000000011", + "10000000000012", + ], + "product_name": [ + "Widget", + "Free Sample", + "Café & Tea", + "茶", + "Book", + "Lamp", + "Chair", + "Desk", + "Glass", + "Mug", + "Bag", + "Pencil", + "Shoes", + "Coat", + "Hat", + "Socks", + ], + "quantity": [1, 1, 2, 1, 1, -2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + "price": [ + "12.50", + "0.00", + "1.234,56", + "₹1,23,456.70", + "9.99", + "10.00", + "11.00", + "12.00", + "13.00", + "14.00", + "15.00", + "16.00", + "17.00", + "18.00", + "19.00", + "20.00", + ], + "currency": [ + "USD", + "USD", + "EUR", + "INR", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + ], + "review": ["great"] * 16, + "email": ["ordinary@example.invalid"] * 16, + "card": ["TB-RETAIL-CARD-0001"] * 16, + "order_date": ["2026-01-15"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "retail", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "locales": ["en_US", "de_DE", "hi_IN", "zh_CN"], + "currency": "USD", + }, + protected_columns=("sku", "gtin"), + ) + builder.inject("ret-01", "sku", "001234", Disposition.PRESERVE, family="leading-zero-sku") + builder.inject( + "ret-01", "gtin", "00012345678905", Disposition.PRESERVE, family="leading-zero-gtin" + ) + builder.inject("ret-02", "price", "0.00", Disposition.PRESERVE, family="free-item") + builder.inject("ret-06", "quantity", -2, Disposition.FLAG, family="return-quantity") + builder.inject( + "ret-03", + "price", + "1.234,56", + Disposition.REPAIR, + expected=1234.56, + expected_dtype="float64", + family="mixed-decimal-grouping", + ) + builder.inject("ret-04", "price", "₹1,23,456.70", Disposition.REVIEW, family="mixed-currency") + builder.inject("ret-05", "currency", "EUR", Disposition.REVIEW, family="mixed-currency") + builder.inject( + "ret-03", + "product_name", + "Café & Tea", + Disposition.REVIEW, + family="html-entity-mojibake", + ) + builder.inject( + "ret-04", "product_name", "茶", Disposition.PRESERVE, family="multilingual-product" + ) + builder.inject( + "ret-07", + "review", + "customer@example.invalid", + Disposition.REVIEW, + family="email-in-review", + sensitive=True, + ) + builder.inject( + "ret-08", + "card", + "TB-RETAIL-CARD-REVIEW", + Disposition.REVIEW, + family="card-in-review", + sensitive=True, + ) + builder.inject( + "ret-16", + "email", + "tail@example.invalid", + Disposition.FLAG, + family="tail-email-canary", + sensitive=True, + ) + builder.add_row_case( + "exact-duplicate-ret-02-ret-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-ret-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case( + "type-drifted-quantity", Disposition.REVIEW, family="type-drifted-column" + ) + return builder.build() diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py index 4fe90f5..088607e 100644 --- a/tests/truthbench/test_fixtures.py +++ b/tests/truthbench/test_fixtures.py @@ -8,7 +8,45 @@ from benchmarks.truthbench.fixtures import build_fixture from benchmarks.truthbench.fixtures.base import FixtureBuilder, FixtureError -DOMAINS = ("minimal",) +DOMAINS = ("minimal", "finance", "healthcare", "retail", "crm") + + +@pytest.mark.parametrize("domain", DOMAINS[1:]) +def test_domain_fixture_has_complete_oracle_and_required_dispositions(domain: str) -> None: + fixture = build_fixture(domain, seed=1729) + assert fixture.frame.shape[0] == 16 + assert {cell.disposition for cell in fixture.cells} == set(Disposition) + assert len(fixture.row_cases) >= 2 + assert len(fixture.schema_cases) >= 4 + assert fixture.policy["reference_date"] == "2026-01-15" + assert fixture.policy["timezone"] == "UTC" + + +@pytest.mark.parametrize("domain", DOMAINS[1:]) +def test_domain_fixture_is_byte_deterministic_for_approved_seeds(domain: str) -> None: + for seed in (1729, 2718): + first = build_fixture(domain, seed=seed) + second = build_fixture(domain, seed=seed) + assert first.fixture_hash == second.fixture_hash + assert first.to_dict() == second.to_dict() + + +@pytest.mark.parametrize( + ("domain", "needles"), + [ + ("finance", ("apple", "Apple", "AAPL", "₹1,23,456.70", "01/02/2025")), + ("healthcare", ("98.6", "5000 mcg", "MRN", "FHIR")), + ("retail", ("SKU", "GTIN", "&", "mojibake")), + ("crm", ("apple", "Apple", ".invalid", "SSN")), + ], +) +def test_domain_fixture_contains_adversarial_family_markers( + domain: str, needles: tuple[str, ...] +) -> None: + fixture = build_fixture(domain, seed=1729) + serialized = json.dumps(fixture.to_dict(), ensure_ascii=False).casefold() + for needle in needles: + assert needle.casefold() in serialized @pytest.mark.parametrize("domain", DOMAINS) From 97c618beea8e7b8ae8912929ece840964fda4d7b Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 12:59:11 +0530 Subject: [PATCH 16/40] fix: validate TruthBench healthcare gold cases --- .superpowers/sdd/task-4-report.md | 23 ++ benchmarks/truthbench/fixtures/healthcare.py | 14 +- tests/truthbench/test_fixtures.py | 208 +++++++++++++++++-- 3 files changed, 221 insertions(+), 24 deletions(-) diff --git a/.superpowers/sdd/task-4-report.md b/.superpowers/sdd/task-4-report.md index 5f21935..47a5388 100644 --- a/.superpowers/sdd/task-4-report.md +++ b/.superpowers/sdd/task-4-report.md @@ -61,3 +61,26 @@ All passed. - The legacy `minimal` fixture remains registered for backwards compatibility; the four Task 4 domains are appended in stable order. A later registry task may choose to retire `minimal` once its callers migrate. - Row/schema expectations are metadata cases (the physical frame remains rectangular), consistent with the oracle contract that removed rows/columns cannot have cell labels. +## Review follow-up: healthcare reference codes and content assertions + +### RED + +After review, focused content tests were strengthened to inspect each actual `GoldCell.family`, disposition, and adversarial frame value. The first run exposed four failures: the healthcare rare-code test rejected the placeholder `G rare`; finance, retail, and CRM injection-count assertions correctly counted only non-preserve dispositions rather than all injected cells. + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result before fixes: `4 failed, 24 passed`. + +### GREEN + +Healthcare preserve cases now use values present in the bundled reference sets (`Z79.4`, `F17.210`, and `9843-4`), and tests load those references plus validate ICD/LOINC syntax. Content tests assert the finance, healthcare, retail, and CRM families directly against physical cells; injected-cell counts use non-background families, including preserve injections. + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result: `28 passed`. + +Complete adjacent verification (`tests/truthbench/test_fixtures.py`, `tests/truthbench/test_models_exact.py`, and `tests/truthbench/test_schema.py`) passed. Ruff check/format, mypy, and `git diff --check` also passed. diff --git a/benchmarks/truthbench/fixtures/healthcare.py b/benchmarks/truthbench/fixtures/healthcare.py index 2c898b8..7ddeed2 100644 --- a/benchmarks/truthbench/fixtures/healthcare.py +++ b/benchmarks/truthbench/fixtures/healthcare.py @@ -35,7 +35,7 @@ def build(seed: int = 1729) -> TruthFixture: "R69", "E11.9", "I10", - "G rare", + "Z79.4", "J45.9", "M54.5", "N39.0", @@ -44,7 +44,7 @@ def build(seed: int = 1729) -> TruthFixture: "D50.9", "G43.9", "H25.1", - "C rare", + "F17.210", "B20", "A09", "Z00.0", @@ -53,7 +53,7 @@ def build(seed: int = 1729) -> TruthFixture: "LP21258-6", "2345-7", "718-7", - "rare-LOINC-123", + "9843-4", "4548-4", "8310-5", "2951-2", @@ -62,7 +62,7 @@ def build(seed: int = 1729) -> TruthFixture: "26499-4", "3094-0", "33747-0", - "rare-LOINC-456", + "9843-4", "1751-7", "2823-3", "1975-2", @@ -132,11 +132,9 @@ def build(seed: int = 1729) -> TruthFixture: protected_columns=("mrn", "dob"), ) builder.inject( - "hc-04", "diagnosis_code", "G rare", Disposition.PRESERVE, family="rare-icd-valid" - ) - builder.inject( - "hc-04", "loinc", "rare-LOINC-123", Disposition.PRESERVE, family="rare-loinc-valid" + "hc-04", "diagnosis_code", "Z79.4", Disposition.PRESERVE, family="rare-icd-valid" ) + builder.inject("hc-04", "loinc", "9843-4", Disposition.PRESERVE, family="rare-loinc-valid") builder.inject( "hc-04", "temperature", 98.6, Disposition.REVIEW, family="celsius-fahrenheit-conflict" ) diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py index 088607e..34937c6 100644 --- a/tests/truthbench/test_fixtures.py +++ b/tests/truthbench/test_fixtures.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import re +from pathlib import Path import pandas as pd import pytest @@ -11,6 +13,18 @@ DOMAINS = ("minimal", "finance", "healthcare", "retail", "crm") +def _cell(fixture, row_id: str, column: str): + return next(cell for cell in fixture.cells if cell.row_id == row_id and cell.column == column) + + +def _assert_cell(fixture, row_id: str, column: str, family: str, value, disposition: Disposition): + cell = _cell(fixture, row_id, column) + assert cell.family == family + assert cell.disposition is disposition + assert fixture.frame.at[row_id, column] == value + return cell + + @pytest.mark.parametrize("domain", DOMAINS[1:]) def test_domain_fixture_has_complete_oracle_and_required_dispositions(domain: str) -> None: fixture = build_fixture(domain, seed=1729) @@ -31,22 +45,184 @@ def test_domain_fixture_is_byte_deterministic_for_approved_seeds(domain: str) -> assert first.to_dict() == second.to_dict() -@pytest.mark.parametrize( - ("domain", "needles"), - [ - ("finance", ("apple", "Apple", "AAPL", "₹1,23,456.70", "01/02/2025")), - ("healthcare", ("98.6", "5000 mcg", "MRN", "FHIR")), - ("retail", ("SKU", "GTIN", "&", "mojibake")), - ("crm", ("apple", "Apple", ".invalid", "SSN")), - ], -) -def test_domain_fixture_contains_adversarial_family_markers( - domain: str, needles: tuple[str, ...] -) -> None: - fixture = build_fixture(domain, seed=1729) - serialized = json.dumps(fixture.to_dict(), ensure_ascii=False).casefold() - for needle in needles: - assert needle.casefold() in serialized +def test_finance_content_families_are_labeled_on_actual_cells() -> None: + fixture = build_fixture("finance", seed=1729) + _assert_cell(fixture, "fin-03", "price", "semantic-apple-price", "apple", Disposition.REVIEW) + _assert_cell(fixture, "fin-04", "company", "apple-company", "Apple", Disposition.PRESERVE) + _assert_cell( + fixture, "fin-04", "ticker", "protected-ticker-policy-conflict", "AAPL", Disposition.REVIEW + ) + _assert_cell(fixture, "fin-02", "price", "zero-price", "0.00", Disposition.PRESERVE) + _assert_cell(fixture, "fin-05", "price", "negative-value", -4.5, Disposition.FLAG) + _assert_cell(fixture, "fin-06", "price", "extreme-value", 9999999999.99, Disposition.FLAG) + _assert_cell( + fixture, "fin-07", "price", "indian-grouped-currency", "₹1,23,456.70", Disposition.REPAIR + ) + _assert_cell( + fixture, "fin-10", "trade_date", "ambiguous-date-format", "01/02/2025", Disposition.REVIEW + ) + pii = _assert_cell( + fixture, + "fin-11", + "memo", + "invisible-pii-memo", + "tb.finance+memo@example.invalid", + Disposition.FLAG, + ) + assert pii.sensitive + tail = _assert_cell( + fixture, + "fin-16", + "account_id", + "tail-row-account-canary", + "TB-FIN-ACCOUNT-TAIL", + Disposition.FLAG, + ) + assert tail.sensitive + assert sum(cell.family != "background" for cell in fixture.cells) >= 12 + + +def test_healthcare_content_families_use_valid_rare_reference_codes() -> None: + fixture = build_fixture("healthcare", seed=1729) + rare_icd = _assert_cell( + fixture, "hc-04", "diagnosis_code", "rare-icd-valid", "Z79.4", Disposition.PRESERVE + ) + rare_loinc = _assert_cell( + fixture, "hc-04", "loinc", "rare-loinc-valid", "9843-4", Disposition.PRESERVE + ) + references = ( + Path(__file__).parents[2] / "src" / "freshdata" / "domains" / "healthcare" / "reference" + ) + assert ( + fixture.frame.at["hc-04", "diagnosis_code"] + in json.loads((references / "icd10_common.json").read_text())["codes"] + ) + assert ( + fixture.frame.at["hc-04", "loinc"] + in json.loads((references / "loinc_common.json").read_text())["codes"] + ) + assert re.fullmatch( + r"[A-TV-Z][0-9]{2}(?:\.[0-9]{1,4})?", fixture.frame.at["hc-04", "diagnosis_code"] + ) + assert re.fullmatch(r"[0-9]{1,5}-[0-9]", fixture.frame.at["hc-04", "loinc"]) + assert rare_icd.disposition is rare_loinc.disposition is Disposition.PRESERVE + _assert_cell( + fixture, "hc-04", "temperature", "celsius-fahrenheit-conflict", 98.6, Disposition.REVIEW + ) + _assert_cell(fixture, "hc-01", "dose", "dose-unit-valid", "5 mg", Disposition.PRESERVE) + _assert_cell( + fixture, "hc-04", "dose", "mg-mcg-unit-conversion", "5000 mcg", Disposition.REPAIR + ) + _assert_cell(fixture, "hc-05", "event_date", "partial-date", "2025-01", Disposition.REVIEW) + _assert_cell( + fixture, "hc-06", "event_date", "fhir-date", "2025-01-15T12:00:00Z", Disposition.REPAIR + ) + _assert_cell(fixture, "hc-07", "event_date", "impossible-date", "2025-02-30", Disposition.FLAG) + _assert_cell( + fixture, "hc-08", "patient_name", "decomposed-unicode", "Jose\u0301", Disposition.REPAIR + ) + for row_id, value, family in ( + ("hc-09", "TB-HC-PHI-0001", "phi-in-notes"), + ("hc-11", "TB-HC-MRN-NOTE", "mrn-in-notes"), + ("hc-12", "555-0112", "phone-in-notes"), + ): + cell = _assert_cell(fixture, row_id, "notes", family, value, Disposition.FLAG) + assert cell.sensitive + assert sum(cell.family != "background" for cell in fixture.cells) >= 12 + + +def test_retail_content_families_are_labeled_on_actual_cells_and_schema_cases() -> None: + fixture = build_fixture("retail", seed=1729) + _assert_cell(fixture, "ret-01", "sku", "leading-zero-sku", "001234", Disposition.PRESERVE) + _assert_cell( + fixture, "ret-01", "gtin", "leading-zero-gtin", "00012345678905", Disposition.PRESERVE + ) + _assert_cell(fixture, "ret-02", "price", "free-item", "0.00", Disposition.PRESERVE) + _assert_cell(fixture, "ret-06", "quantity", "return-quantity", -2, Disposition.FLAG) + _assert_cell( + fixture, "ret-03", "price", "mixed-decimal-grouping", "1.234,56", Disposition.REPAIR + ) + _assert_cell(fixture, "ret-04", "price", "mixed-currency", "₹1,23,456.70", Disposition.REVIEW) + _assert_cell( + fixture, + "ret-03", + "product_name", + "html-entity-mojibake", + "Café & Tea", + Disposition.REVIEW, + ) + _assert_cell( + fixture, "ret-04", "product_name", "multilingual-product", "茶", Disposition.PRESERVE + ) + for row_id, column, value, family in ( + ("ret-07", "review", "customer@example.invalid", "email-in-review"), + ("ret-08", "card", "TB-RETAIL-CARD-REVIEW", "card-in-review"), + ): + cell = _assert_cell(fixture, row_id, column, family, value, Disposition.REVIEW) + assert cell.sensitive + assert {case.family for case in fixture.schema_cases} >= { + "added-column", + "removed-column", + "renamed-column", + "reordered-columns", + "type-drifted-column", + } + assert sum(cell.family != "background" for cell in fixture.cells) >= 12 + + +def test_crm_content_families_are_labeled_on_actual_cells() -> None: + fixture = build_fixture("crm", seed=1729) + _assert_cell( + fixture, "crm-02", "first_name", "combining-unicode-name", "Jose\u0301", Disposition.REPAIR + ) + _assert_cell( + fixture, + "crm-03", + "email", + "reserved-invalid-email", + "miyuki@example.invalid", + Disposition.PRESERVE, + ) + _assert_cell(fixture, "crm-04", "phone", "spaced-phone", "555 0101", Disposition.REPAIR) + _assert_cell(fixture, "crm-05", "country", "ambiguous-country", "US/CA", Disposition.REVIEW) + _assert_cell(fixture, "crm-06", "language", "ambiguous-language", "en/fr", Disposition.REVIEW) + _assert_cell( + fixture, "crm-07", "signup_date", "ambiguous-date", "01/02/2025", Disposition.REVIEW + ) + _assert_cell( + fixture, + "crm-08", + "lifecycle", + "lifecycle-contradiction", + "lead|churned", + Disposition.REVIEW, + ) + _assert_cell( + fixture, + "crm-11", + "email", + "zero-width-email", + "lead\u200b@example.invalid", + Disposition.FLAG, + ) + ssn = _assert_cell( + fixture, "crm-10", "notes", "hidden-ssn", "TB-CRM-SSN-0001", Disposition.FLAG + ) + assert ssn.sensitive + _assert_cell( + fixture, "crm-03", "lead_source", "apple-lead-source", "apple", Disposition.PRESERVE + ) + _assert_cell(fixture, "crm-03", "employer", "apple-employer", "Apple", Disposition.PRESERVE) + protected = _assert_cell( + fixture, + "crm-16", + "customer_id", + "protected-customer-id", + "TB-CRM-CUSTOMER-TAIL", + Disposition.REVIEW, + ) + assert protected.sensitive + assert sum(cell.family != "background" for cell in fixture.cells) >= 12 @pytest.mark.parametrize("domain", DOMAINS) From da40954103467e0e4383c5e40c9d63b179e0cc2a Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 13:04:23 +0530 Subject: [PATCH 17/40] test: close TruthBench family coverage gaps --- .superpowers/sdd/task-4-report.md | 24 ++++++++ benchmarks/truthbench/fixtures/finance.py | 2 +- tests/truthbench/test_fixtures.py | 68 +++++++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/.superpowers/sdd/task-4-report.md b/.superpowers/sdd/task-4-report.md index 47a5388..7418575 100644 --- a/.superpowers/sdd/task-4-report.md +++ b/.superpowers/sdd/task-4-report.md @@ -84,3 +84,27 @@ PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov Result: `28 passed`. Complete adjacent verification (`tests/truthbench/test_fixtures.py`, `tests/truthbench/test_models_exact.py`, and `tests/truthbench/test_schema.py`) passed. Ruff check/format, mypy, and `git diff --check` also passed. + +## Review follow-up: contract-gap family coverage + +### RED + +Added direct assertions for the finance USD/EUR/INR conflict and zero-width memo, healthcare protected-DOB repair conflict and MRN tail canary, and exact row/schema family sets for finance, healthcare, and CRM. The initial run exposed the missing `zero-width-memo` family label (the value existed but was grouped under the broader invisible-PII family). + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result: `1 failed, 30 passed` (`StopIteration` while locating the required zero-width memo family). + +### GREEN + +The zero-width memo cell now has its own `zero-width-memo` family; all assertions inspect actual frame values, dispositions, sensitivity, and exact case-family sets. + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result: `31 passed`. + +Complete fixture/models/schema verification passed (`... passed`), as did Ruff check/format, mypy, and `git diff --check`. diff --git a/benchmarks/truthbench/fixtures/finance.py b/benchmarks/truthbench/fixtures/finance.py index 598f5d0..511b41c 100644 --- a/benchmarks/truthbench/fixtures/finance.py +++ b/benchmarks/truthbench/fixtures/finance.py @@ -182,7 +182,7 @@ def build(seed: int = 1729) -> TruthFixture: "memo", "tb.finance+hidden@example.invalid\u200b", Disposition.FLAG, - family="invisible-pii-memo", + family="zero-width-memo", ) builder.inject( "fin-16", diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py index 34937c6..5dab7e4 100644 --- a/tests/truthbench/test_fixtures.py +++ b/tests/truthbench/test_fixtures.py @@ -82,6 +82,32 @@ def test_finance_content_families_are_labeled_on_actual_cells() -> None: assert sum(cell.family != "background" for cell in fixture.cells) >= 12 +def test_finance_currency_conflict_zero_width_memo_and_exact_case_families() -> None: + fixture = build_fixture("finance", seed=1729) + currencies = { + fixture.frame.at[cell.row_id, cell.column] + for cell in fixture.cells + if cell.family == "usd-eur-inr-conflict" + } + assert currencies == {"EUR", "INR"} + assert {fixture.frame.at[row, "currency"] for row in fixture.frame.index} >= { + "USD", + "EUR", + "INR", + } + zero_width = next(cell for cell in fixture.cells if cell.family == "zero-width-memo") + assert "\u200b" in fixture.frame.at[zero_width.row_id, zero_width.column] + assert zero_width.disposition is Disposition.FLAG + assert {case.family for case in fixture.row_cases} == {"exact-duplicate", "removed-row"} + assert {case.family for case in fixture.schema_cases} == { + "added-column", + "removed-column", + "renamed-column", + "reordered-columns", + "type-drifted-column", + } + + def test_healthcare_content_families_use_valid_rare_reference_codes() -> None: fixture = build_fixture("healthcare", seed=1729) rare_icd = _assert_cell( @@ -131,6 +157,36 @@ def test_healthcare_content_families_use_valid_rare_reference_codes() -> None: assert sum(cell.family != "background" for cell in fixture.cells) >= 12 +def test_healthcare_protected_dob_tail_and_exact_case_families() -> None: + fixture = build_fixture("healthcare", seed=1729) + dob = _assert_cell( + fixture, + "hc-15", + "dob", + "protected-dob-repair-conflict", + "01/01/1980", + Disposition.REVIEW, + ) + tail = _assert_cell( + fixture, + "hc-16", + "mrn", + "mrn-tail-canary", + "TB-HC-MRN-TAIL", + Disposition.REVIEW, + ) + assert dob.column in fixture.protected_columns + assert tail.sensitive + assert {case.family for case in fixture.row_cases} == {"exact-duplicate", "removed-row"} + assert {case.family for case in fixture.schema_cases} == { + "added-column", + "removed-column", + "renamed-column", + "reordered-columns", + "type-drifted-column", + } + + def test_retail_content_families_are_labeled_on_actual_cells_and_schema_cases() -> None: fixture = build_fixture("retail", seed=1729) _assert_cell(fixture, "ret-01", "sku", "leading-zero-sku", "001234", Disposition.PRESERVE) @@ -225,6 +281,18 @@ def test_crm_content_families_are_labeled_on_actual_cells() -> None: assert sum(cell.family != "background" for cell in fixture.cells) >= 12 +def test_crm_has_exact_row_and_schema_case_family_sets() -> None: + fixture = build_fixture("crm", seed=1729) + assert {case.family for case in fixture.row_cases} == {"exact-duplicate", "removed-row"} + assert {case.family for case in fixture.schema_cases} == { + "added-column", + "removed-column", + "renamed-column", + "reordered-columns", + "type-drifted-column", + } + + @pytest.mark.parametrize("domain", DOMAINS) def test_every_physical_cell_has_exactly_one_label(domain: str) -> None: fixture = build_fixture(domain, seed=1729) From 502f437d4cf5bb39b0abb07883db3e43e361784b Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 13:17:32 +0530 Subject: [PATCH 18/40] feat: complete eight-domain TruthBench corpus --- .superpowers/sdd/task-5-report.md | 85 ++++ benchmarks/truthbench/fixtures/__init__.py | 43 +-- benchmarks/truthbench/fixtures/education.py | 187 +++++++++ benchmarks/truthbench/fixtures/government.py | 170 ++++++++ benchmarks/truthbench/fixtures/insurance.py | 208 ++++++++++ benchmarks/truthbench/fixtures/logistics.py | 172 +++++++++ tests/truthbench/test_fixtures.py | 385 ++++++++++++++++++- 7 files changed, 1217 insertions(+), 33 deletions(-) create mode 100644 .superpowers/sdd/task-5-report.md create mode 100644 benchmarks/truthbench/fixtures/education.py create mode 100644 benchmarks/truthbench/fixtures/government.py create mode 100644 benchmarks/truthbench/fixtures/insurance.py create mode 100644 benchmarks/truthbench/fixtures/logistics.py diff --git a/.superpowers/sdd/task-5-report.md b/.superpowers/sdd/task-5-report.md new file mode 100644 index 0000000..6ad9f93 --- /dev/null +++ b/.superpowers/sdd/task-5-report.md @@ -0,0 +1,85 @@ +# Task 5 Report: Complete eight-domain TruthBench corpus + +## Status + +DONE. Logistics, government, education, and insurance now have deterministic 16-row gold fixtures. The registry is the stable alphabetical eight-domain order (`crm`, `education`, `finance`, `government`, `healthcare`, `insurance`, `logistics`, `retail`) with the temporary minimal registry removed. + +## TDD evidence + +### RED + +After adding domain-content tests, the focused fixture suite failed with the expected missing-builder errors for all four new domains (`FixtureError: unknown fixture domain`). The failure run was: + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +### GREEN + +The focused suite now passes: + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result: `47 passed`. + +Adjacent fixture/model/schema verification: + +```text +PYTHONPATH=src python -m pytest \ + tests/truthbench/test_fixtures.py \ + tests/truthbench/test_models_exact.py \ + tests/truthbench/test_schema.py -q --no-cov +``` + +Result: `112 passed`. + +Static checks: + +```text +python -m ruff check benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py +python -m ruff format --check benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py +mypy --ignore-missing-imports --explicit-package-bases \ + benchmarks/truthbench/fixtures tests/truthbench/test_fixtures.py +git diff --check +``` + +All passed. + +## Coverage + +- `logistics.py` covers valid UN/LOCODE-like references, kg/lb and C/F units, cross-timezone windows, 24:00 transport values, address PII, late tracking, and protected shipment IDs. +- `government.py` covers leading-zero IDs, Indian/international grouping, fiscal/calendar ambiguity, multilingual labels, restricted national IDs, mixed legacy encoding, and retention/repair policy contradiction. +- `education.py` covers student IDs, letter/percentage/GPA scales, school-year ambiguity, zero scores, enrollment ordering, guardian contacts, FERPA notes, and protected grade-policy conflict. +- `insurance.py` covers policy/claim IDs, premium/reserve currency mismatch, negative reserve review, incident/report ordering, state contradiction, claimant/medical PII, and protected policy numbers. +- All four builders emit complete physical-cell labels, all four dispositions, row duplicate/removal cases, five schema drift cases, fixed UTC/reference metadata, deterministic seed batches, and privacy-safe synthetic canaries. +- Corpus-level tests assert all required trap categories occur across the eight domains. + +## Concerns + +None. No FreshData runtime, LLM/provider, network, or external data dependency is used. + +## Review follow-up: explicit contract coverage + +### RED + +The new required-family contract test intentionally used the repaired numeric output (`95`) as the adversarial frame value for education `edu-07`. The focused test failed because the actual frame value is the required raw value `"95%"` while the repair oracle separately stores `95.0` as its expected output. + +```text +PYTHONPATH=src python -m pytest \ + tests/truthbench/test_fixtures.py::test_required_domain_families_match_actual_values_and_dispositions \ + -q --no-cov +``` + +Observed: one failure at `edu-07` (`'95%' != 95`). + +### GREEN + +The contract now asserts the raw value, family, disposition, and typed repair output. It also uses an explicit per-domain family mapping covering every required category (including logistics lb/F units, government IDs/grouping/language/fiscal/protected case, education scales/contacts/protected grade, and insurance IDs/grouped premium/PII/protected policy). + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_fixtures.py -q --no-cov +``` + +Result: `48 passed`. diff --git a/benchmarks/truthbench/fixtures/__init__.py b/benchmarks/truthbench/fixtures/__init__.py index 704c334..8ed5936 100644 --- a/benchmarks/truthbench/fixtures/__init__.py +++ b/benchmarks/truthbench/fixtures/__init__.py @@ -4,13 +4,20 @@ from typing import Callable -import pandas as pd - -from . import crm, finance, healthcare, retail +from . import crm, education, finance, government, healthcare, insurance, logistics, retail from .base import FixtureBuilder, FixtureError, TruthFixture _BUILDERS: dict[str, Callable[[int], TruthFixture]] = {} -DOMAINS: tuple[str, ...] = ("minimal", "finance", "healthcare", "retail", "crm") +DOMAINS: tuple[str, ...] = ( + "crm", + "education", + "finance", + "government", + "healthcare", + "insurance", + "logistics", + "retail", +) def register_fixture(domain: str, builder: Callable[[int], TruthFixture]) -> None: @@ -21,32 +28,14 @@ def register_fixture(domain: str, builder: Callable[[int], TruthFixture]) -> Non _BUILDERS[domain] = builder -def _minimal(seed: int) -> TruthFixture: - frame = pd.DataFrame( - { - "name": ["alpha", "beta"], - "amount": [float(seed % 10), float((seed % 10) + 1)], - }, - index=["r1", "r2"], - ) - builder = FixtureBuilder( - "v1", - "minimal", - frame, - seed=seed, - schema={"columns": ["name", "amount"]}, - policy={"locale": "en_US"}, - protected_columns=("name",), - ) - builder.inject("r2", "amount", "2.50", "repair", expected=2.5, family="numeric-format") - return builder.build() - - -register_fixture("minimal", _minimal) +register_fixture("crm", crm.build) +register_fixture("education", education.build) register_fixture("finance", finance.build) +register_fixture("government", government.build) register_fixture("healthcare", healthcare.build) +register_fixture("insurance", insurance.build) +register_fixture("logistics", logistics.build) register_fixture("retail", retail.build) -register_fixture("crm", crm.build) def build_fixture(domain: str, seed: int = 1729) -> TruthFixture: diff --git a/benchmarks/truthbench/fixtures/education.py b/benchmarks/truthbench/fixtures/education.py new file mode 100644 index 0000000..039a055 --- /dev/null +++ b/benchmarks/truthbench/fixtures/education.py @@ -0,0 +1,187 @@ +"""Deterministic education gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"edu-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "student_id": [ + "000123", + "000124", + "000125", + "000126", + "000127", + "000128", + "000129", + "000130", + "000131", + "000132", + "000133", + "000134", + "000135", + "000136", + "000137", + "000138", + ], + "grade_letter": [ + "A", + "A-", + "B+", + "B", + "C+", + "C", + "B-", + "A", + "A", + "B", + "C", + "D", + "A", + "B+", + "C", + "A", + ], + "score_percent": [ + "95", + "90", + "87.5", + "82", + "78", + "72", + "68", + "100", + "0", + "88", + "76", + "61", + "93", + "86", + "74", + "96", + ], + "gpa": [ + 4.0, + 3.7, + 3.3, + 3.0, + 2.7, + 2.3, + 2.0, + 4.0, + 0.0, + 3.3, + 2.7, + 1.7, + 3.9, + 3.3, + 2.3, + 4.0, + ], + "school_year": ["2025-2026"] * 16, + "enrollment_date": ["2025-08-15"] * 16, + "completion_date": ["2026-05-30"] * 16, + "guardian_email": ["guardian1@example.invalid"] * 16, + "guardian_phone": ["555-0101"] * 16, + "ferpa_notes": ["routine"] * 16, + "grade_policy": ["A>=90;B>=80;C>=70"] * 16, + "language": ["en"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "education", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "grade_scales": ["letter", "percentage", "GPA"], + "protected_grade_policy": "preserve", + }, + protected_columns=("student_id", "grade_letter"), + ) + builder.inject( + "edu-01", "student_id", "000123", Disposition.PRESERVE, family="leading-zero-student-id" + ) + builder.inject( + "edu-02", "grade_letter", "A-", Disposition.PRESERVE, family="letter-grade-scale" + ) + builder.inject("edu-03", "score_percent", 0, Disposition.PRESERVE, family="zero-score") + builder.inject( + "edu-04", "school_year", "2025/26", Disposition.REVIEW, family="school-year-ambiguity" + ) + builder.inject( + "edu-05", + "enrollment_date", + "2026-02-01", + Disposition.REVIEW, + family="enrollment-date-ordering", + ) + builder.inject( + "edu-06", + "completion_date", + "2025-01-01", + Disposition.REVIEW, + family="enrollment-date-ordering", + ) + builder.inject( + "edu-07", + "score_percent", + "95%", + Disposition.REPAIR, + expected=95.0, + expected_dtype="float64", + family="percentage-scale", + ) + builder.inject("edu-08", "gpa", 4.0, Disposition.PRESERVE, family="gpa-scale") + builder.inject( + "edu-09", + "guardian_email", + "guardian@example.invalid", + Disposition.FLAG, + family="guardian-contact-pii", + sensitive=True, + ) + builder.inject( + "edu-10", + "guardian_phone", + "555-0110", + Disposition.FLAG, + family="guardian-contact-pii", + sensitive=True, + ) + builder.inject( + "edu-10", + "ferpa_notes", + "TB-EDU-FERPA-0001", + Disposition.FLAG, + family="ferpa-sensitive-notes", + sensitive=True, + ) + builder.inject( + "edu-16", "grade_letter", "A", Disposition.REVIEW, family="protected-grade-policy-conflict" + ) + builder.add_row_case( + "exact-duplicate-edu-02-edu-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-edu-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case("type-drifted-score", Disposition.REVIEW, family="type-drifted-column") + return builder.build() diff --git a/benchmarks/truthbench/fixtures/government.py b/benchmarks/truthbench/fixtures/government.py new file mode 100644 index 0000000..14dc0e2 --- /dev/null +++ b/benchmarks/truthbench/fixtures/government.py @@ -0,0 +1,170 @@ +"""Deterministic government gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"gov-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "district_id": [ + "001", + "002", + "003", + "004", + "005", + "006", + "007", + "008", + "009", + "010", + "011", + "012", + "013", + "014", + "015", + "016", + ], + "case_id": [ + "000001", + "000123", + "000003", + "000004", + "000005", + "000006", + "000007", + "000008", + "000009", + "000010", + "000011", + "000012", + "000013", + "000014", + "000015", + "000016", + ], + "agency": [ + "Revenue Department", + "भारत सरकार / Government of India", + "Ministry of Health", + "City Council", + "税務局", + "Prefecture Office", + "State Secretariat", + "Public Works", + "Education Board", + "Transport Authority", + "Treasury", + "Registry", + "Civil Court", + "Labor Office", + "Statistics Bureau", + "Revenue Department", + ], + "fiscal_year": ["2026"] * 16, + "calendar_year": ["2026"] * 16, + "language": ["en"] * 16, + "notes": ["routine"] * 16, + "encoding": ["UTF-8"] * 16, + "retention_policy": ["retain 7 years"] * 16, + "repair_policy": ["repair after 30 days"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "government", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "supported_languages": ["en", "hi", "de", "zh"], + "protected_case_policy": "preserve", + }, + protected_columns=("district_id", "case_id"), + ) + builder.inject( + "gov-01", "district_id", "007", Disposition.PRESERVE, family="leading-zero-district-id" + ) + builder.inject( + "gov-02", "case_id", "000123", Disposition.PRESERVE, family="leading-zero-case-id" + ) + builder.inject( + "gov-03", + "agency", + "भारत सरकार / Government of India", + Disposition.PRESERVE, + family="indian-international-grouping", + ) + builder.inject( + "gov-05", + "fiscal_year", + "2025-26", + Disposition.REVIEW, + family="fiscal-calendar-year-conflict", + ) + builder.inject( + "gov-06", + "encoding", + "Café", + Disposition.REPAIR, + expected="Café", + family="mixed-legacy-encoding", + ) + builder.inject( + "gov-07", "language", "हिन्दी / English", Disposition.PRESERVE, family="multilingual-agency" + ) + builder.inject( + "gov-09", + "notes", + "TB-GOV-NATIONAL-ID-0001", + Disposition.FLAG, + family="restricted-national-id", + sensitive=True, + ) + builder.inject( + "gov-11", + "retention_policy", + "retain 7 years", + Disposition.REVIEW, + family="contradictory-retention-repair-policy", + ) + builder.inject( + "gov-11", + "repair_policy", + "repair after 30 days", + Disposition.REVIEW, + family="contradictory-retention-repair-policy", + ) + builder.inject( + "gov-16", + "case_id", + "TB-GOV-CASE-TAIL", + Disposition.REVIEW, + family="protected-case-id-conflict", + sensitive=True, + ) + builder.add_row_case( + "exact-duplicate-gov-02-gov-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-gov-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case( + "type-drifted-fiscal-year", Disposition.REVIEW, family="type-drifted-column" + ) + return builder.build() diff --git a/benchmarks/truthbench/fixtures/insurance.py b/benchmarks/truthbench/fixtures/insurance.py new file mode 100644 index 0000000..25ce595 --- /dev/null +++ b/benchmarks/truthbench/fixtures/insurance.py @@ -0,0 +1,208 @@ +"""Deterministic insurance gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"ins-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "policy_number": [ + "00012345", + "00012346", + "00012347", + "00012348", + "00012349", + "00012350", + "00012351", + "00012352", + "00012353", + "00012354", + "00012355", + "00012356", + "00012357", + "00012358", + "00012359", + "00012360", + ], + "claim_id": [ + "CLM-000123", + "CLM-000124", + "CLM-000125", + "CLM-000126", + "CLM-000127", + "CLM-000128", + "CLM-000129", + "CLM-000130", + "CLM-000131", + "CLM-000132", + "CLM-000133", + "CLM-000134", + "CLM-000135", + "CLM-000136", + "CLM-000137", + "CLM-000138", + ], + "premium": [1000.0] * 16, + "premium_currency": ["USD"] * 16, + "reserve": [ + 500.0, + 600.0, + 700.0, + 800.0, + -250.0, + 900.0, + 1000.0, + 1100.0, + 1200.0, + 1300.0, + 1400.0, + 1500.0, + 1600.0, + 1700.0, + 1800.0, + 1900.0, + ], + "reserve_currency": [ + "USD", + "USD", + "USD", + "EUR", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + "USD", + ], + "incident_date": ["2025-01-01"] * 16, + "report_date": ["2025-01-02"] * 16, + "state": [ + "open", + "closed", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + "open", + ], + "claimant_name": ["Claimant"] * 16, + "loss_description": ["water damage"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "insurance", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "currency": "USD", + "supported_currencies": ["USD", "EUR", "INR"], + "protected_policy_policy": "preserve", + }, + protected_columns=("policy_number",), + ) + builder.inject( + "ins-01", "policy_number", "00012345", Disposition.PRESERVE, family="policy-id-format" + ) + builder.inject( + "ins-02", "claim_id", "CLM-000123", Disposition.PRESERVE, family="claim-id-format" + ) + builder.inject( + "ins-03", + "premium", + "1,000.00", + Disposition.REPAIR, + expected=1000.0, + expected_dtype="float64", + family="grouped-premium", + ) + builder.inject( + "ins-04", + "reserve_currency", + "EUR", + Disposition.REVIEW, + family="premium-reserve-currency-conflict", + ) + builder.inject( + "ins-05", "reserve", -250.0, Disposition.REVIEW, family="negative-reserve-review" + ) + builder.inject( + "ins-06", + "report_date", + "2025-01-01", + Disposition.REVIEW, + family="incident-report-date-ordering", + ) + builder.inject( + "ins-07", + "state", + "open|closed", + Disposition.REVIEW, + family="state-transition-contradiction", + ) + builder.inject( + "ins-09", + "claimant_name", + "TB-INS-CLAIMANT-0001", + Disposition.FLAG, + family="claimant-pii", + sensitive=True, + ) + builder.inject( + "ins-10", + "loss_description", + "TB-INS-MEDICAL-0001", + Disposition.FLAG, + family="medical-loss-text", + sensitive=True, + ) + builder.inject( + "ins-16", + "policy_number", + "TB-INS-POLICY-TAIL", + Disposition.REVIEW, + family="protected-policy-number-conflict", + sensitive=True, + ) + builder.add_row_case( + "exact-duplicate-ins-02-ins-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-ins-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case( + "type-drifted-reserve", Disposition.REVIEW, family="type-drifted-column" + ) + return builder.build() diff --git a/benchmarks/truthbench/fixtures/logistics.py b/benchmarks/truthbench/fixtures/logistics.py new file mode 100644 index 0000000..db58c96 --- /dev/null +++ b/benchmarks/truthbench/fixtures/logistics.py @@ -0,0 +1,172 @@ +"""Deterministic logistics gold fixture for TruthBench.""" + +from __future__ import annotations + +import pandas as pd + +from ..models import Disposition +from .base import FixtureBuilder, TruthFixture + + +def build(seed: int = 1729) -> TruthFixture: + rows = [f"log-{i:02d}" for i in range(1, 17)] + frame = pd.DataFrame( + { + "shipment_id": [f"TB-LOG-{i:04d}" for i in range(1, 17)], + "origin_code": ["USLAX"] * 16, + "destination_code": [ + "NLRTM", + "DEHAM", + "GBFXT", + "INBOM", + "SGSIN", + "CNSHA", + "JPTYO", + "AUMEL", + "BRSSZ", + "ZADUR", + "AEJEA", + "NOOSL", + "ESBCN", + "CATOR", + "MYPKG", + "INBOM", + ], + "weight": [ + 10.0, + "10 lb", + 12.0, + 8.0, + 4.0, + 15.0, + 20.0, + 25.0, + 30.0, + 35.0, + 40.0, + 45.0, + 50.0, + 55.0, + 60.0, + 65.0, + ], + "weight_unit": ["kg"] * 16, + "temperature": [ + 20.0, + 21.0, + 98.6, + 19.0, + 18.0, + 22.0, + 23.0, + 24.0, + 25.0, + 26.0, + 27.0, + 28.0, + 29.0, + 30.0, + 31.0, + 32.0, + ], + "temperature_unit": ["C"] * 16, + "delivery_window": ["2026-01-15 09:00-17:00"] * 16, + "transport_time": ["12:00"] * 16, + "address": ["Warehouse"] * 16, + "tracking_status": ["on-time"] * 16, + "timezone": ["UTC"] * 16, + "batch": [f"seed-{int(seed)}"] * 16, + }, + index=rows, + ) + builder = FixtureBuilder( + "v1", + "logistics", + frame, + seed=int(seed), + schema={ + "columns": list(frame.columns), + "dtypes": {c: str(frame[c].dtype) for c in frame.columns}, + }, + policy={ + "reference_date": "2026-01-15", + "timezone": "UTC", + "locale": "en_US", + "weight_unit": "kg", + "temperature_unit": "C", + "protected_shipment_policy": "preserve", + }, + protected_columns=("shipment_id",), + ) + builder.inject( + "log-04", "destination_code", "INBOM", Disposition.PRESERVE, family="rare-unlocode-valid" + ) + builder.inject( + "log-02", + "weight", + "10 lb", + Disposition.REPAIR, + expected=4.5359237, + expected_dtype="float64", + family="kg-lb-conversion", + ) + builder.inject("log-02", "weight_unit", "lb", Disposition.PRESERVE, family="weight-unit-lb") + builder.inject( + "log-03", "temperature", 98.6, Disposition.REVIEW, family="celsius-fahrenheit-conflict" + ) + builder.inject( + "log-03", "temperature_unit", "F", Disposition.PRESERVE, family="temperature-unit-f" + ) + builder.inject( + "log-05", + "delivery_window", + "2026-01-15 23:30-2026-01-16 01:00", + Disposition.REVIEW, + family="cross-timezone-window", + ) + builder.inject( + "log-05", + "timezone", + "Asia/Kolkata→America/New_York", + Disposition.REVIEW, + family="cross-timezone-window", + ) + builder.inject( + "log-06", + "transport_time", + "24:00", + Disposition.REPAIR, + expected="00:00", + family="twentyfour-hour-transport", + ) + builder.inject( + "log-09", + "address", + "TB-LOG-ADDRESS-0001", + Disposition.FLAG, + family="address-pii", + sensitive=True, + ) + builder.inject( + "log-15", "tracking_status", "delayed", Disposition.FLAG, family="late-tracking-canary" + ) + builder.inject( + "log-16", + "shipment_id", + "TB-LOG-SHIPMENT-TAIL", + Disposition.REVIEW, + family="protected-shipment-id-conflict", + sensitive=True, + ) + builder.add_row_case( + "exact-duplicate-log-02-log-03", Disposition.FLAG, family="exact-duplicate" + ) + builder.add_row_case("removed-log-15", Disposition.REVIEW, family="removed-row") + builder.add_schema_case("added-column", Disposition.REVIEW, family="added-column") + builder.add_schema_case("removed-column", Disposition.REVIEW, family="removed-column") + builder.add_schema_case("renamed-column", Disposition.REVIEW, family="renamed-column") + builder.add_schema_case("reordered-columns", Disposition.REVIEW, family="reordered-columns") + builder.add_schema_case( + "type-drifted-weight", Disposition.REVIEW, family="type-drifted-column" + ) + return builder.build() diff --git a/tests/truthbench/test_fixtures.py b/tests/truthbench/test_fixtures.py index 5dab7e4..842c6ee 100644 --- a/tests/truthbench/test_fixtures.py +++ b/tests/truthbench/test_fixtures.py @@ -10,7 +10,16 @@ from benchmarks.truthbench.fixtures import build_fixture from benchmarks.truthbench.fixtures.base import FixtureBuilder, FixtureError -DOMAINS = ("minimal", "finance", "healthcare", "retail", "crm") +DOMAINS = ( + "crm", + "education", + "finance", + "government", + "healthcare", + "insurance", + "logistics", + "retail", +) def _cell(fixture, row_id: str, column: str): @@ -25,7 +34,7 @@ def _assert_cell(fixture, row_id: str, column: str, family: str, value, disposit return cell -@pytest.mark.parametrize("domain", DOMAINS[1:]) +@pytest.mark.parametrize("domain", DOMAINS) def test_domain_fixture_has_complete_oracle_and_required_dispositions(domain: str) -> None: fixture = build_fixture(domain, seed=1729) assert fixture.frame.shape[0] == 16 @@ -36,7 +45,7 @@ def test_domain_fixture_has_complete_oracle_and_required_dispositions(domain: st assert fixture.policy["timezone"] == "UTC" -@pytest.mark.parametrize("domain", DOMAINS[1:]) +@pytest.mark.parametrize("domain", DOMAINS) def test_domain_fixture_is_byte_deterministic_for_approved_seeds(domain: str) -> None: for seed in (1729, 2718): first = build_fixture(domain, seed=seed) @@ -293,6 +302,370 @@ def test_crm_has_exact_row_and_schema_case_family_sets() -> None: } +def test_logistics_content_families_are_labeled_on_actual_cells() -> None: + fixture = build_fixture("logistics", seed=1729) + _assert_cell( + fixture, "log-04", "destination_code", "rare-unlocode-valid", "INBOM", Disposition.PRESERVE + ) + _assert_cell(fixture, "log-02", "weight", "kg-lb-conversion", "10 lb", Disposition.REPAIR) + _assert_cell(fixture, "log-02", "weight_unit", "weight-unit-lb", "lb", Disposition.PRESERVE) + _assert_cell( + fixture, "log-03", "temperature", "celsius-fahrenheit-conflict", 98.6, Disposition.REVIEW + ) + _assert_cell( + fixture, "log-03", "temperature_unit", "temperature-unit-f", "F", Disposition.PRESERVE + ) + _assert_cell( + fixture, + "log-05", + "delivery_window", + "cross-timezone-window", + "2026-01-15 23:30-2026-01-16 01:00", + Disposition.REVIEW, + ) + _assert_cell( + fixture, + "log-05", + "timezone", + "cross-timezone-window", + "Asia/Kolkata→America/New_York", + Disposition.REVIEW, + ) + _assert_cell( + fixture, + "log-06", + "transport_time", + "twentyfour-hour-transport", + "24:00", + Disposition.REPAIR, + ) + for row_id, column, family in ( + ("log-09", "address", "address-pii"), + ("log-15", "tracking_status", "late-tracking-canary"), + ): + cell = _assert_cell( + fixture, row_id, column, family, fixture.frame.at[row_id, column], Disposition.FLAG + ) + if family == "address-pii": + assert cell.sensitive + protected = _assert_cell( + fixture, + "log-16", + "shipment_id", + "protected-shipment-id-conflict", + "TB-LOG-SHIPMENT-TAIL", + Disposition.REVIEW, + ) + assert protected.sensitive + + +def test_government_content_families_are_labeled_on_actual_cells() -> None: + fixture = build_fixture("government", seed=1729) + _assert_cell( + fixture, "gov-01", "district_id", "leading-zero-district-id", "007", Disposition.PRESERVE + ) + _assert_cell( + fixture, "gov-02", "case_id", "leading-zero-case-id", "000123", Disposition.PRESERVE + ) + _assert_cell( + fixture, + "gov-03", + "agency", + "indian-international-grouping", + "भारत सरकार / Government of India", + Disposition.PRESERVE, + ) + _assert_cell( + fixture, + "gov-05", + "fiscal_year", + "fiscal-calendar-year-conflict", + "2025-26", + Disposition.REVIEW, + ) + _assert_cell( + fixture, + "gov-07", + "language", + "multilingual-agency", + "हिन्दी / English", + Disposition.PRESERVE, + ) + _assert_cell( + fixture, "gov-06", "encoding", "mixed-legacy-encoding", "Café", Disposition.REPAIR + ) + restricted = _assert_cell( + fixture, + "gov-09", + "notes", + "restricted-national-id", + "TB-GOV-NATIONAL-ID-0001", + Disposition.FLAG, + ) + assert restricted.sensitive + _assert_cell( + fixture, + "gov-11", + "retention_policy", + "contradictory-retention-repair-policy", + "retain 7 years", + Disposition.REVIEW, + ) + tail = _assert_cell( + fixture, + "gov-16", + "case_id", + "protected-case-id-conflict", + "TB-GOV-CASE-TAIL", + Disposition.REVIEW, + ) + assert tail.sensitive + + +def test_education_content_families_are_labeled_on_actual_cells() -> None: + fixture = build_fixture("education", seed=1729) + _assert_cell( + fixture, "edu-01", "student_id", "leading-zero-student-id", "000123", Disposition.PRESERVE + ) + _assert_cell( + fixture, "edu-02", "grade_letter", "letter-grade-scale", "A-", Disposition.PRESERVE + ) + _assert_cell(fixture, "edu-03", "score_percent", "zero-score", 0, Disposition.PRESERVE) + _assert_cell( + fixture, "edu-04", "school_year", "school-year-ambiguity", "2025/26", Disposition.REVIEW + ) + _assert_cell( + fixture, + "edu-05", + "enrollment_date", + "enrollment-date-ordering", + "2026-02-01", + Disposition.REVIEW, + ) + _assert_cell(fixture, "edu-08", "gpa", "gpa-scale", 4.0, Disposition.PRESERVE) + guardian = _assert_cell( + fixture, + "edu-09", + "guardian_email", + "guardian-contact-pii", + "guardian@example.invalid", + Disposition.FLAG, + ) + assert guardian.sensitive + phone = _assert_cell( + fixture, + "edu-10", + "guardian_phone", + "guardian-contact-pii", + "555-0110", + Disposition.FLAG, + ) + assert phone.sensitive + ferpa = _assert_cell( + fixture, + "edu-10", + "ferpa_notes", + "ferpa-sensitive-notes", + "TB-EDU-FERPA-0001", + Disposition.FLAG, + ) + assert ferpa.sensitive + protected = _assert_cell( + fixture, + "edu-16", + "grade_letter", + "protected-grade-policy-conflict", + "A", + Disposition.REVIEW, + ) + assert protected.column in fixture.protected_columns + + +def test_insurance_content_families_are_labeled_on_actual_cells() -> None: + fixture = build_fixture("insurance", seed=1729) + _assert_cell( + fixture, "ins-01", "policy_number", "policy-id-format", "00012345", Disposition.PRESERVE + ) + _assert_cell( + fixture, "ins-02", "claim_id", "claim-id-format", "CLM-000123", Disposition.PRESERVE + ) + _assert_cell( + fixture, + "ins-04", + "reserve_currency", + "premium-reserve-currency-conflict", + "EUR", + Disposition.REVIEW, + ) + _assert_cell( + fixture, "ins-05", "reserve", "negative-reserve-review", -250.0, Disposition.REVIEW + ) + _assert_cell( + fixture, + "ins-06", + "report_date", + "incident-report-date-ordering", + "2025-01-01", + Disposition.REVIEW, + ) + _assert_cell( + fixture, + "ins-07", + "state", + "state-transition-contradiction", + "open|closed", + Disposition.REVIEW, + ) + claimant = _assert_cell( + fixture, + "ins-09", + "claimant_name", + "claimant-pii", + "TB-INS-CLAIMANT-0001", + Disposition.FLAG, + ) + assert claimant.sensitive + medical = _assert_cell( + fixture, + "ins-10", + "loss_description", + "medical-loss-text", + "TB-INS-MEDICAL-0001", + Disposition.FLAG, + ) + assert medical.sensitive + protected = _assert_cell( + fixture, + "ins-16", + "policy_number", + "protected-policy-number-conflict", + "TB-INS-POLICY-TAIL", + Disposition.REVIEW, + ) + assert protected.sensitive + + +def test_eight_domain_corpus_contains_required_trap_categories() -> None: + expected_families = { + "logistics": { + "rare-unlocode-valid", + "kg-lb-conversion", + "weight-unit-lb", + "celsius-fahrenheit-conflict", + "temperature-unit-f", + "cross-timezone-window", + "twentyfour-hour-transport", + "address-pii", + "late-tracking-canary", + "protected-shipment-id-conflict", + }, + "government": { + "leading-zero-district-id", + "leading-zero-case-id", + "indian-international-grouping", + "multilingual-agency", + "fiscal-calendar-year-conflict", + "restricted-national-id", + "mixed-legacy-encoding", + "contradictory-retention-repair-policy", + "protected-case-id-conflict", + }, + "education": { + "leading-zero-student-id", + "letter-grade-scale", + "percentage-scale", + "gpa-scale", + "school-year-ambiguity", + "zero-score", + "enrollment-date-ordering", + "guardian-contact-pii", + "ferpa-sensitive-notes", + "protected-grade-policy-conflict", + }, + "insurance": { + "policy-id-format", + "claim-id-format", + "grouped-premium", + "premium-reserve-currency-conflict", + "negative-reserve-review", + "incident-report-date-ordering", + "state-transition-contradiction", + "claimant-pii", + "medical-loss-text", + "protected-policy-number-conflict", + }, + } + for domain, required in expected_families.items(): + actual = { + cell.family + for cell in build_fixture(domain, seed=1729).cells + if cell.family and cell.family != "background" + } + assert required <= actual + + +def test_required_domain_families_match_actual_values_and_dispositions() -> None: + logistics = build_fixture("logistics", seed=1729) + for row, column, family, logistics_value, disposition in ( + ("log-02", "weight", "kg-lb-conversion", "10 lb", Disposition.REPAIR), + ("log-02", "weight_unit", "weight-unit-lb", "lb", Disposition.PRESERVE), + ("log-03", "temperature_unit", "temperature-unit-f", "F", Disposition.PRESERVE), + ): + _assert_cell(logistics, row, column, family, logistics_value, disposition) + + government = build_fixture("government", seed=1729) + for row, column, family, government_value, disposition in ( + ("gov-01", "district_id", "leading-zero-district-id", "007", Disposition.PRESERVE), + ("gov-02", "case_id", "leading-zero-case-id", "000123", Disposition.PRESERVE), + ( + "gov-03", + "agency", + "indian-international-grouping", + "भारत सरकार / Government of India", + Disposition.PRESERVE, + ), + ("gov-07", "language", "multilingual-agency", "हिन्दी / English", Disposition.PRESERVE), + ("gov-05", "fiscal_year", "fiscal-calendar-year-conflict", "2025-26", Disposition.REVIEW), + ( + "gov-16", + "case_id", + "protected-case-id-conflict", + "TB-GOV-CASE-TAIL", + Disposition.REVIEW, + ), + ): + _assert_cell(government, row, column, family, government_value, disposition) + + education = build_fixture("education", seed=1729) + for row, column, family, education_value, disposition in ( + ("edu-01", "student_id", "leading-zero-student-id", "000123", Disposition.PRESERVE), + ("edu-02", "grade_letter", "letter-grade-scale", "A-", Disposition.PRESERVE), + ("edu-07", "score_percent", "percentage-scale", "95%", Disposition.REPAIR), + ("edu-08", "gpa", "gpa-scale", 4.0, Disposition.PRESERVE), + ("edu-16", "grade_letter", "protected-grade-policy-conflict", "A", Disposition.REVIEW), + ): + _assert_cell(education, row, column, family, education_value, disposition) + percentage = _cell(education, "edu-07", "score_percent") + assert percentage.expected_output is not None + assert percentage.expected_output.display == "95.0" + + insurance = build_fixture("insurance", seed=1729) + for row, column, family, value, disposition in ( + ("ins-01", "policy_number", "policy-id-format", "00012345", Disposition.PRESERVE), + ("ins-02", "claim_id", "claim-id-format", "CLM-000123", Disposition.PRESERVE), + ("ins-03", "premium", "grouped-premium", "1,000.00", Disposition.REPAIR), + ("ins-09", "claimant_name", "claimant-pii", "TB-INS-CLAIMANT-0001", Disposition.FLAG), + ( + "ins-16", + "policy_number", + "protected-policy-number-conflict", + "TB-INS-POLICY-TAIL", + Disposition.REVIEW, + ), + ): + _assert_cell(insurance, row, column, family, value, disposition) + + @pytest.mark.parametrize("domain", DOMAINS) def test_every_physical_cell_has_exactly_one_label(domain: str) -> None: fixture = build_fixture(domain, seed=1729) @@ -373,10 +746,10 @@ def test_repair_expected_output_cannot_be_contradictory() -> None: def test_fixture_hash_is_stable_and_includes_metadata() -> None: - first = build_fixture("minimal", seed=1729) - second = build_fixture("minimal", seed=1729) + first = build_fixture("finance", seed=1729) + second = build_fixture("finance", seed=1729) assert first.fixture_hash == second.fixture_hash altered = FixtureBuilder( - "v1", "minimal", first.pristine.copy(), schema={"different": True} + "v1", "finance", first.pristine.copy(), schema={"different": True} ).build() assert altered.fixture_hash != first.fixture_hash From 7255974419f2df2db19c6f2745f8ca120055c599 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 13:53:12 +0530 Subject: [PATCH 19/40] feat: add TruthBench PII leak scanner --- .superpowers/sdd/task-6-report.md | 153 +++++++++ benchmarks/truthbench/__init__.py | 6 + benchmarks/truthbench/privacy.py | 541 ++++++++++++++++++++++++++++++ tests/truthbench/test_privacy.py | 160 +++++++++ 4 files changed, 860 insertions(+) create mode 100644 .superpowers/sdd/task-6-report.md create mode 100644 benchmarks/truthbench/privacy.py create mode 100644 tests/truthbench/test_privacy.py diff --git a/.superpowers/sdd/task-6-report.md b/.superpowers/sdd/task-6-report.md new file mode 100644 index 0000000..ca3911a --- /dev/null +++ b/.superpowers/sdd/task-6-report.md @@ -0,0 +1,153 @@ +# Task 6 Report: Privacy-safe values and exhaustive sink scanning + +## Status + +DONE. `SinkScanner` now scans normalized canary variants across nested TruthBench +sinks and emits only `Leak(canary_id, variant, path)` metadata. Redaction markers +contain run-scoped HMAC-SHA256 digests and never matched text. + +## TDD evidence + +### RED + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_privacy.py -q --no-cov +``` + +Observed collection failure before implementation: + +```text +ModuleNotFoundError: No module named 'benchmarks.truthbench.privacy' +``` + +### GREEN + +Focused privacy suite: + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_privacy.py -q --no-cov +``` + +Result: `18 passed`. + +Adjacent TruthBench suites: + +```text +PYTHONPATH=src python -m pytest \ + tests/truthbench/test_privacy.py \ + tests/truthbench/test_fixtures.py \ + tests/truthbench/test_models_exact.py \ + tests/truthbench/test_schema.py -q --no-cov +``` + +Result: `131 passed`. + +Static checks: + +```text +python -m ruff check benchmarks/truthbench/privacy.py \ + benchmarks/truthbench/__init__.py tests/truthbench/test_privacy.py +python -m ruff format --check benchmarks/truthbench/privacy.py \ + benchmarks/truthbench/__init__.py tests/truthbench/test_privacy.py +mypy --ignore-missing-imports --explicit-package-bases \ + benchmarks/truthbench/privacy.py tests/truthbench/test_privacy.py +git diff --check +``` + +All passed. + +## Files + +- `benchmarks/truthbench/privacy.py` — `Leak`, `PrivacySafeValue`, named + normalizers, run-scoped HMAC scanner, recursive redaction, typed redaction + handling, pandas/dataclass/bytes support, self-test, and named sink entry points. +- `benchmarks/truthbench/__init__.py` — exports privacy scanner primitives. +- `tests/truthbench/test_privacy.py` — mutation matrix for every required + normalized form, nested sink coverage, redaction/self-test behavior, and typed + redaction digest safety. + +## Self-review + +- Leak objects contain only identifiers, transform labels, and JSONPath-like + locations; `repr(leaks)` cannot repeat canary text. +- Literal, case-folded, whitespace-stripped, punctuation-stripped, digit-only, + URL-decoded, HTML-unescaped, UTF-8/hex/escape bytes, NFKC/NFC/NFD, + zero-width-removed, and JSON-escaped forms are covered. +- Mapping, sequence, dataclass, pandas DataFrame/Series/Index, exception, + report, plan, generated-code, stream, markup, JSON, and failure-artifact sinks + are traversed. Exact redacted `TypedValue` payloads are treated as safe and + their HMAC digests are not re-scanned as plaintext. +- Redaction is recursive, emits `[REDACTED:]`, and `self_test` raises on + an unredacted result while accepting the scanner's own redacted output. + +## Concerns + +- The scanner intentionally treats digit-only normalized forms conservatively; + very short numeric canaries can match unrelated text. Fixtures use synthetic + identifiers and email/phone canaries, so this does not affect the bundled + corpus. +- Redacting a pandas object may change a sensitive column to object/string dtype, + which is preferable to retaining a raw value in an audit sink. + +## Follow-up RED/GREEN evidence + +A follow-up regression test used the exact sensitive `TypedValue` redaction shape +with a one-digit canary. Before the redacted-payload guard, the digest's hex text +was incorrectly reported as a digit-only leak. After adding the guard: + +```text +PYTHONPATH=src python -m pytest \ + tests/truthbench/test_privacy.py::test_scanner_accepts_exact_typed_redaction_without_scanning_digest \ + -q --no-cov +``` + +Result: `1 passed`. + +## Review follow-up: marker, key, and label hardening + +### RED + +The review regression matrix was run before the hardening changes: + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_privacy.py -q --no-cov +``` + +Result: `4 failed, 17 passed` for forged redaction markers, sensitive mapping +keys, pandas labels, and the digest compatibility alias. + +### GREEN + +After validating markers against the scanner's own 64-hex HMAC digest set, +redacting mapping keys and pandas column/index/name labels, scanning arbitrary +key stringifications, and adding `digest` as an alias: + +```text +PYTHONPATH=src python -m pytest tests/truthbench/test_privacy.py -q --no-cov +``` + +Result: `23 passed`. + +Adjacent TruthBench verification: + +```text +PYTHONPATH=src python -m pytest \ + tests/truthbench/test_privacy.py \ + tests/truthbench/test_fixtures.py \ + tests/truthbench/test_models_exact.py \ + tests/truthbench/test_schema.py -q --no-cov +``` + +Result: `136 passed`. + +Static checks were rerun after the follow-up and remained clean: + +```text +python -m ruff check benchmarks/truthbench/privacy.py \ + tests/truthbench/test_privacy.py +python -m ruff format --check benchmarks/truthbench/privacy.py \ + tests/truthbench/test_privacy.py +mypy --ignore-missing-imports --explicit-package-bases \ + benchmarks/truthbench/privacy.py tests/truthbench/test_privacy.py +git diff --check +``` diff --git a/benchmarks/truthbench/__init__.py b/benchmarks/truthbench/__init__.py index b3e08a1..d2e1690 100644 --- a/benchmarks/truthbench/__init__.py +++ b/benchmarks/truthbench/__init__.py @@ -10,6 +10,7 @@ GoldCell, RunResult, ) +from .privacy import Leak, PrivacySafeValue, SinkScanner, normalize_variants, redact_value __all__ = [ "UNSET", @@ -24,4 +25,9 @@ "encode_typed", "exact_equal", "stable_digest", + "Leak", + "PrivacySafeValue", + "SinkScanner", + "normalize_variants", + "redact_value", ] diff --git a/benchmarks/truthbench/privacy.py b/benchmarks/truthbench/privacy.py new file mode 100644 index 0000000..22f0371 --- /dev/null +++ b/benchmarks/truthbench/privacy.py @@ -0,0 +1,541 @@ +"""Privacy-safe TruthBench sink values and exhaustive canary scanning. + +The scanner deliberately stores only canary identifiers, transform names, paths, +and run-scoped HMAC digests. Matched input is never included in a ``Leak`` or +in a redaction marker. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import hmac +import html +import json +import re +import unicodedata +import urllib.parse +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Callable + +import pandas as pd + +from .exact import canonical_json, encode_typed + + +@dataclass(frozen=True) +class Leak: + """A privacy leak without retaining the matched value.""" + + canary_id: str + variant: str + path: str + + +@dataclass(frozen=True) +class PrivacySafeValue: + """A leaf value represented by a redacted display and keyed digest.""" + + value: str = "[REDACTED]" + digest: str = "" + + def __repr__(self) -> str: + return f"PrivacySafeValue(value={self.value!r}, digest={self.digest!r})" + + +_ZERO_WIDTH = "\u200b\u200c\u200d\ufeff" + + +def _remove_zero_width(value: str) -> str: + return "".join( + char for char in value if char not in _ZERO_WIDTH and unicodedata.category(char) != "Cf" + ) + + +def _remove_punctuation(value: str) -> str: + return "".join(char for char in value if not unicodedata.category(char).startswith("P")) + + +def _json_escaped(value: str) -> str: + encoded = json.dumps(value, ensure_ascii=True) + return encoded[1:-1] + + +def _digit_only(value: str) -> str: + return "".join(char for char in value if char.isdigit()) + + +Transform = Callable[[str], str] + +# Public names make the normalization contract inspectable and testable. +NORMALIZERS: tuple[tuple[str, Transform], ...] = ( + ("literal", lambda value: value), + ("casefold", str.casefold), + ("whitespace-stripped", lambda value: re.sub(r"\s+", "", value)), + ("punctuation-stripped", _remove_punctuation), + ("digit-only", _digit_only), + ("url-decoded", urllib.parse.unquote), + ("html-unescaped", html.unescape), + ("nfkc", lambda value: unicodedata.normalize("NFKC", value)), + ("nfc", lambda value: unicodedata.normalize("NFC", value)), + ("nfd", lambda value: unicodedata.normalize("NFD", value)), + ("zero-width-removed", _remove_zero_width), + ("json-escaped", _json_escaped), +) + + +def normalize_variants(value: str) -> dict[str, str]: + """Return named normalized forms of text, omitting empty forms.""" + + if not isinstance(value, str): + raise TypeError("normalize_variants expects text") + result: dict[str, str] = {} + for name, transform in NORMALIZERS: + try: + normalized = transform(value) + except (UnicodeError, ValueError): + continue + if normalized: + result.setdefault(name, normalized) + return result + + +def _text_forms(value: Any) -> dict[str, str]: + """Convert text/bytes to forms which can be safely searched.""" + + if isinstance(value, str): + return normalize_variants(value) + if isinstance(value, bytes): + forms: dict[str, str] = {} + try: + decoded = value.decode("utf-8") + except UnicodeDecodeError: + decoded = "" + if decoded: + for name, text in normalize_variants(decoded).items(): + forms.setdefault(name, text) + forms["bytes-hex"] = value.hex() + forms["bytes-escape"] = value.decode("latin1") + forms["bytes-repr"] = repr(value) + return forms + return {} + + +def _path_component(path: str, key: Any) -> str: + key_text = str(key) + if key_text.isidentifier(): + return f"{path}.{key_text}" + return f"{path}[{json.dumps(key_text, ensure_ascii=False)}]" + + +def _is_redacted_typed_mapping(value: Mapping[Any, Any]) -> bool: + """Recognize the exact redacted ``TypedValue.to_dict`` shape.""" + + return ( + value.get("redacted") is True + and value.get("value") is None + and value.get("display") == "[REDACTED]" + and bool(value.get("digest")) + ) + + +_DIGEST_MARKER = re.compile(r"^\[REDACTED:([0-9a-f]{64})\]$") + + +class SinkScanner: + """Scan arbitrary nested sink values for normalized synthetic canaries.""" + + def __init__(self, canaries: Mapping[str, Any], *, key: bytes, run_id: str = "run") -> None: + if not isinstance(canaries, Mapping) or not canaries: + raise ValueError("at least one canary is required") + if not isinstance(key, bytes) or not key: + raise TypeError("key must be non-empty bytes") + self.run_id = run_id + self._key = key + self._canaries = {str(identifier): value for identifier, value in canaries.items()} + self._digests = { + identifier: self._digest(value) for identifier, value in self._canaries.items() + } + self._patterns: dict[str, tuple[tuple[str, str], ...]] = {} + for identifier, value in self._canaries.items(): + forms: list[tuple[str, str]] = [] + if isinstance(value, bytes): + forms.extend(_text_forms(value).items()) + elif isinstance(value, str): + forms.extend(normalize_variants(value).items()) + forms.append(("utf8", value.encode("utf-8").decode("latin1"))) + forms.append(("bytes-hex", value.encode("utf-8").hex())) + forms.append( + ("bytes-escape", "".join(f"\\x{byte:02x}" for byte in value.encode("utf-8"))) + ) + else: + forms.extend(normalize_variants(str(value)).items()) + # Keep longest forms first; digit-only canaries otherwise produce + # surprising matches before the literal form. + priority = {name: index for index, (name, _) in enumerate(NORMALIZERS)} + priority.update( + {"utf8": 100, "bytes-hex": 101, "bytes-escape": 102, "bytes-repr": 103} + ) + self._patterns[identifier] = tuple( + sorted( + set(forms), + key=lambda pair: (-len(pair[1]), priority.get(pair[0], 200)), + ) + ) + + @classmethod + def from_canaries( + cls, + canaries: Mapping[str, Any], + *, + run_id: str = "run", + key: bytes | None = None, + run_key: bytes | None = None, + ) -> SinkScanner: + """Build a scanner with a run-scoped HMAC-SHA256 key.""" + + if key is not None and run_key is not None: + raise ValueError("pass only one of key or run_key") + if run_key is not None: + key = run_key + if key is None: + key = hmac.new( + b"truthbench-privacy-run-v1", + str(run_id).encode("utf-8"), + hashlib.sha256, + ).digest() + return cls(canaries, key=key, run_id=run_id) + + @classmethod + def from_fixture( + cls, + fixture: Any, + *, + run_id: str = "run", + key: bytes | None = None, + ) -> SinkScanner: + """Build directly from a ``TruthFixture`` canary mapping.""" + + canaries = getattr(fixture, "pii_canaries", None) + if not isinstance(canaries, Mapping): + raise TypeError("fixture must expose a pii_canaries mapping") + return cls.from_canaries(canaries, run_id=run_id, key=key) + + def _digest(self, value: Any) -> str: + encoded = canonical_json(encode_typed(value)).encode("utf-8") + return hmac.new(self._key, encoded, hashlib.sha256).hexdigest() + + @property + def digests(self) -> Mapping[str, str]: + return dict(self._digests) + + def _scan_text(self, value: Any, path: str) -> Leak | None: + if ( + isinstance(value, str) + and (match := _DIGEST_MARKER.fullmatch(value)) is not None + and match.group(1) in self._digests.values() + ): + return None + forms = _text_forms(value) + for identifier in self._canaries: + if isinstance(value, bytes): + byte_patterns = { + variant: pattern + for variant, pattern in self._patterns[identifier] + if variant in {"utf8", "bytes-escape", "bytes-hex", "bytes-repr"} + } + for variant in ("utf8", "bytes-escape", "bytes-hex", "bytes-repr"): + pattern = byte_patterns.get(variant) + if pattern is not None and any(pattern in text for text in forms.values()): + return Leak(identifier, variant, path) + for variant, pattern in self._patterns[identifier]: + if not pattern: + continue + same_form = forms.get(variant) + if same_form is not None and pattern in same_form: + return Leak(identifier, variant, path) + # Byte and escape forms do not have corresponding normalizers + # on ordinary text, so retain a conservative cross-form check. + if variant.startswith("bytes-") or variant == "utf8": + for text in forms.values(): + if pattern in text: + return Leak(identifier, variant, path) + return None + + def scan(self, sink: Any, *, path: str = "$") -> list[Leak]: + """Recursively scan a sink and return one leak per canary/path.""" + + leaks: list[Leak] = [] + seen: set[tuple[str, str]] = set() + + def visit(value: Any, current_path: str) -> None: + if isinstance(value, (str, bytes)): + leak = self._scan_text(value, current_path) + if leak is not None and (leak.canary_id, leak.path) not in seen: + seen.add((leak.canary_id, leak.path)) + leaks.append(leak) + return + if value is None or isinstance(value, (bool, int, float, complex)): + return + if isinstance(value, Mapping) and _is_redacted_typed_mapping(value): + return + if ( + getattr(value, "redacted", False) is True + and getattr(value, "value", object()) is None + and getattr(value, "display", None) == "[REDACTED]" + ): + return + if isinstance(value, pd.DataFrame): + for column in value.columns: + visit(column, f"{current_path}[]") + for index in value.index: + visit(index, f"{current_path}[]") + for column in value.columns: + column_path = ( + f"{current_path}[]" + if self._scan_text(column, current_path) + else _path_component(current_path, column) + ) + for index, cell in value[column].items(): + visit( + cell, + f"{column_path}[]" + if self._scan_text(index, current_path) + else ( + f"{column_path}[{index!r}]" + if isinstance(index, str) + else f"{column_path}[{index}]" + ), + ) + return + if isinstance(value, pd.Series): + if value.name is not None: + visit(value.name, f"{current_path}[]") + for index in value.index: + visit(index, f"{current_path}[]") + for index, cell in value.items(): + visit( + cell, + f"{current_path}[]" + if self._scan_text(index, current_path) + else ( + f"{current_path}[{index!r}]" + if isinstance(index, str) + else f"{current_path}[{index}]" + ), + ) + return + if isinstance(value, pd.Index): + if value.name is not None: + visit(value.name, f"{current_path}[]") + for index, cell in enumerate(value): + visit(cell, f"{current_path}[{index}]") + return + if dataclasses.is_dataclass(value) and not isinstance(value, type): + for field in dataclasses.fields(value): + visit(getattr(value, field.name), _path_component(current_path, field.name)) + return + if isinstance(value, Mapping): + for key, item in value.items(): + # Keys can also be serialized sinks, but never include a + # raw key in a reported path. + try: + key_text = key if isinstance(key, (str, bytes)) else str(key) + except Exception: # pragma: no cover - hostile key object + key_text = f"<{type(key).__name__}>" + visit(key_text, f"{current_path}[]") + key_leak = self._scan_text(key_text, current_path) + item_path = ( + f"{current_path}[]" + if key_leak is not None + else _path_component(current_path, key) + ) + visit(item, item_path) + return + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + for index, item in enumerate(value): + visit(item, f"{current_path}[{index}]") + return + to_dict = getattr(value, "to_dict", None) + if callable(to_dict): + try: + visit(to_dict(), current_path) + return + except Exception: # pragma: no cover - defensive sink handling + pass + attrs = getattr(value, "__dict__", None) + if isinstance(attrs, Mapping): + visit(attrs, current_path) + + visit(sink, path) + return leaks + + def redact(self, sink: Any) -> Any: + """Return a recursively redacted copy of a sink value.""" + + def redact_value(value: Any) -> Any: + if isinstance(value, (str, bytes)): + leak = self._scan_text(value, "$") + if leak is not None: + # A digest marker contains no input-derived text. + return f"[REDACTED:{self._digests[leak.canary_id]}]" + return value + if isinstance(value, pd.DataFrame): + copied = value.copy(deep=True) + copied.columns = [redact_value(column) for column in copied.columns] + copied.index = pd.Index([redact_value(index) for index in copied.index]) + for column in copied.columns: + copied[column] = copied[column].map(redact_value) + return copied + if isinstance(value, pd.Series): + copied = value.map(redact_value) + copied.index = pd.Index([redact_value(index) for index in copied.index]) + copied.name = redact_value(value.name) if value.name is not None else None + return copied + if isinstance(value, pd.Index): + return pd.Index( + [redact_value(item) for item in value], + name=redact_value(value.name) if value.name is not None else None, + ) + if isinstance(value, Mapping) and _is_redacted_typed_mapping(value): + return dict(value) + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: redact_value(getattr(value, field.name)) + for field in dataclasses.fields(value) + } + if isinstance(value, Mapping): + result: dict[Any, Any] = {} + for key, item in value.items(): + try: + key_text = key if isinstance(key, (str, bytes)) else str(key) + except Exception: # pragma: no cover - hostile key object + key_text = f"<{type(key).__name__}>" + key_leak = self._scan_text(key_text, "$") + safe_key = ( + f"[REDACTED:{self._digests[key_leak.canary_id]}]" + if key_leak is not None + else key + ) + result[safe_key] = redact_value(item) + return result + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [redact_value(item) for item in value] + to_dict = getattr(value, "to_dict", None) + if callable(to_dict): + try: + return redact_value(to_dict()) + except Exception: # pragma: no cover + pass + return value + + return redact_value(sink) + + def self_test(self, sink: Any) -> bool: + leaks = self.scan(sink) + if leaks: + raise AssertionError("privacy self-test found one or more canary leaks") + return True + + def scan_sinks(self, sinks: Mapping[str, Any]) -> list[Leak]: + return self.scan(sinks) + + def scan_all(self, **sinks: Any) -> list[Leak]: + """Scan all supplied named output sinks in one pass.""" + + return self.scan(sinks) + + def digest_for(self, value: Any) -> str: + """Return a run-scoped digest without retaining the value.""" + + return self._digest(value) + + def digest(self, value: Any) -> str: + """Compatibility alias for :meth:`digest_for`.""" + + return self.digest_for(value) + + def redact_with_digests(self, sink: Any) -> Any: + """Alias emphasizing that redaction markers contain keyed digests.""" + + return self.redact(sink) + + def redact_value(self, value: Any) -> Any: + """Method alias for callers that redact one sink value.""" + + return self.redact(value) + + def _scan_named(self, name: str, value: Any) -> list[Leak]: + return self.scan(value, path=f"$.{name}") + + # Named entry points keep sink coverage explicit for callers and audits. + def scan_exception(self, value: Any) -> list[Leak]: + return self.scan(value, path="$.exception_text") + + def scan_clean_report(self, value: Any) -> list[Leak]: + return self.scan(value, path="$.clean_report") + + def scan_action_metadata(self, value: Any) -> list[Leak]: + return self.scan(value, path="$.actions") + + def scan_generated_code(self, value: Any) -> list[Leak]: + return self.scan(value, path="$.generated_code") + + def scan_output(self, value: Any) -> list[Leak]: + return self.scan(value, path="$.output") + + def scan_failure_artifacts(self, value: Any) -> list[Leak]: + return self.scan(value, path="$.failure_artifacts") + + def scan_coerced_cells(self, value: Any) -> list[Leak]: + return self._scan_named("coerced_cells", value) + + def scan_findings(self, value: Any) -> list[Leak]: + return self._scan_named("findings", value) + + def scan_plan(self, value: Any) -> list[Leak]: + return self._scan_named("plan", value) + + def scan_validation_report(self, value: Any) -> list[Leak]: + return self._scan_named("validation", value) + + def scan_domain_report(self, value: Any) -> list[Leak]: + return self._scan_named("domain", value) + + def scan_privacy_report(self, value: Any) -> list[Leak]: + return self._scan_named("privacy", value) + + def scan_copilot_report(self, value: Any) -> list[Leak]: + return self._scan_named("copilot", value) + + def scan_stdout(self, value: Any) -> list[Leak]: + return self._scan_named("stdout", value) + + def scan_stderr(self, value: Any) -> list[Leak]: + return self._scan_named("stderr", value) + + def scan_markdown(self, value: Any) -> list[Leak]: + return self._scan_named("markdown", value) + + def scan_html(self, value: Any) -> list[Leak]: + return self._scan_named("html", value) + + def scan_json(self, value: Any) -> list[Leak]: + return self._scan_named("json", value) + + +def redact_value(value: Any, scanner: SinkScanner) -> Any: + """Functional convenience wrapper around :meth:`SinkScanner.redact`.""" + + return scanner.redact(value) + + +__all__ = [ + "Leak", + "NORMALIZERS", + "PrivacySafeValue", + "SinkScanner", + "normalize_variants", + "redact_value", +] diff --git a/tests/truthbench/test_privacy.py b/tests/truthbench/test_privacy.py new file mode 100644 index 0000000..961424f --- /dev/null +++ b/tests/truthbench/test_privacy.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import dataclasses +import html +import json +import re +import unicodedata +import urllib.parse + +import pandas as pd +import pytest +from benchmarks.truthbench.exact import encode_typed +from benchmarks.truthbench.privacy import Leak, SinkScanner + +CANARY = "tb.person+7@example.invalid" + + +def _url_quote(value: str) -> str: + return urllib.parse.quote(value) + + +def _html_escape(value: str) -> str: + return html.escape(value) + + +def _json_dump(value: str) -> str: + return json.dumps(value) + + +def _mutators() -> list: + return [ + lambda s: s, + lambda s: s.casefold(), + lambda s: re.sub(r"\s+", "", s), + lambda s: re.sub(r"[^\w\s]", "", s), + lambda s: re.sub(r"\D", "", s), + _url_quote, + _html_escape, + lambda s: s.encode("utf-8"), + lambda s: unicodedata.normalize("NFKC", s), + lambda s: unicodedata.normalize("NFC", s), + lambda s: unicodedata.normalize("NFD", s), + lambda s: s.replace(".", "\u200b.\u200b"), + _json_dump, + ] + + +@pytest.mark.parametrize("mutate", _mutators()) +def test_scanner_finds_every_normalized_variant(mutate): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + leaks = scanner.scan({"report": [mutate(CANARY)]}) + assert [(leak.canary_id, leak.path) for leak in leaks] == [("crm-email", "$.report[0]")] + assert CANARY not in repr(leaks) + + +def test_scan_nested_dataclass_pandas_and_bytes_without_raw_values(): + @dataclasses.dataclass + class Envelope: + exception_text: str + frame: pd.DataFrame + + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + sink = Envelope("failed for " + CANARY, pd.DataFrame({"email": [CANARY.encode()]})) + leaks = scanner.scan(sink) + assert {leak.path for leak in leaks} == { + "$.exception_text", + "$.frame.email[0]", + } + assert all(CANARY not in repr(leak) for leak in leaks) + + +def test_redact_returns_digest_marker_and_self_test(): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}, run_id="run-1") + redacted = scanner.redact({"email": CANARY, "nested": [CANARY]}) + assert CANARY not in repr(redacted) + assert scanner.scan(redacted) == [] + assert scanner.self_test(redacted) is True + with pytest.raises(AssertionError): + scanner.self_test({"email": CANARY}) + + +def test_scanner_accepts_exact_typed_redaction_without_scanning_digest(): + scanner = SinkScanner.from_canaries({"short": "7"}, run_id="run-1") + typed = encode_typed("7", sensitive=True, digest_key=b"test-key").to_dict() + assert scanner.scan(typed) == [] + + +def test_mapping_keys_are_scanned_without_echoing_sensitive_key(): + scanner = SinkScanner.from_canaries({"short": "7"}) + leaks = scanner.scan({"7": "safe"}) + assert [(leak.canary_id, leak.path) for leak in leaks] == [("short", "$[]")] + assert "'7'" not in repr(leaks) + + +def test_arbitrary_mapping_key_stringification_is_sanitized(): + class SensitiveKey: + def __str__(self): + return CANARY + + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + sink = {SensitiveKey(): "safe"} + assert scanner.scan(sink)[0].path == "$[]" + redacted = scanner.redact(sink) + assert CANARY not in repr(redacted) + + +def test_forged_redaction_marker_is_not_trusted(): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + forged = "[REDACTED:" + CANARY + "]" + with pytest.raises(AssertionError): + scanner.self_test(forged) + + +def test_redact_removes_sensitive_mapping_keys(): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + redacted = scanner.redact({CANARY: "safe"}) + assert CANARY not in repr(redacted) + assert scanner.scan(redacted) == [] + + +def test_dataframe_and_series_labels_are_scanned_and_redacted(): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + frame = pd.DataFrame({CANARY: ["safe"]}, index=[CANARY]) + series = pd.Series(["safe"], index=[CANARY], name=CANARY) + assert scanner.scan(frame) + assert scanner.scan(series) + assert scanner.scan(pd.Index([CANARY])) + assert scanner.scan(pd.Index(["safe"], name=CANARY)) + assert scanner.scan(pd.Series(name=CANARY)) + redacted_frame = scanner.redact(frame) + redacted_series = scanner.redact(series) + assert CANARY not in repr(redacted_frame) + assert CANARY not in repr(redacted_series) + + +def test_digest_alias_matches_digest_for(): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + assert scanner.digest(CANARY) == scanner.digest_for(CANARY) + + +def test_named_sink_scanners_cover_report_surfaces(): + scanner = SinkScanner.from_canaries({"crm-email": CANARY}) + sinks = { + "exception_text": CANARY, + "clean_report": {"coerced_cells": {"email": CANARY}}, + "actions": [{"metadata": CANARY}], + "findings": [{"observed_value": CANARY}], + "plan": {"json": CANARY}, + "reports": {"validation": CANARY, "copilot": CANARY}, + "generated_code": CANARY, + "stdout": CANARY, + "stderr": CANARY, + "markdown": CANARY, + "html": CANARY, + "json": CANARY, + "failure_artifacts": [CANARY], + } + leaks = scanner.scan_sinks(sinks) + assert len(leaks) == 14 + assert all(isinstance(leak, Leak) for leak in leaks) From 49c22e1cded51248c7444bb837014f95c7e544c4 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Tue, 14 Jul 2026 14:56:38 +0530 Subject: [PATCH 20/40] fix: harden MultiIndex privacy redaction --- .superpowers/sdd/task-6-report.md | 49 ++++++++ benchmarks/truthbench/privacy.py | 187 ++++++++++++++++++++++++++---- tests/truthbench/test_privacy.py | 50 ++++++++ 3 files changed, 262 insertions(+), 24 deletions(-) diff --git a/.superpowers/sdd/task-6-report.md b/.superpowers/sdd/task-6-report.md index ca3911a..ae4f228 100644 --- a/.superpowers/sdd/task-6-report.md +++ b/.superpowers/sdd/task-6-report.md @@ -54,6 +54,55 @@ mypy --ignore-missing-imports --explicit-package-bases \ git diff --check ``` +## Review follow-up: MultiIndex and hostile-label privacy hardening + +### RED + +Added regression tests for MultiIndex columns/index level names, tuple-label +structure during redaction, and custom non-string labels whose stringification +contains a canary. Before the fix: + +```text +pytest -q tests/truthbench/test_privacy.py --no-cov +``` + +Result: `24 passed, 2 failed`. + +The failures were the expected defects: redacting tuple labels converted them +to lists and raised pandas `ValueError` (length mismatch), while hostile custom +labels were not scanned. + +### GREEN + +Structure-preserving label traversal/redaction now handles tuple/MultiIndex +levels and names, and sanitizes custom label paths before reporting or replacing +them with digest markers: + +```text +pytest -q tests/truthbench/test_privacy.py --no-cov +``` + +Result: `26 passed`. + +Adjacent TruthBench verification: + +```text +pytest -q tests/truthbench --no-cov +``` + +Result: `139 passed`. + +Static checks: + +```text +ruff check benchmarks/truthbench/privacy.py tests/truthbench/test_privacy.py +ruff format --check benchmarks/truthbench/privacy.py tests/truthbench/test_privacy.py +mypy benchmarks/truthbench/privacy.py +git diff --check +``` + +All passed. + All passed. ## Files diff --git a/benchmarks/truthbench/privacy.py b/benchmarks/truthbench/privacy.py index 22f0371..052579d 100644 --- a/benchmarks/truthbench/privacy.py +++ b/benchmarks/truthbench/privacy.py @@ -123,7 +123,14 @@ def _text_forms(value: Any) -> dict[str, str]: def _path_component(path: str, key: Any) -> str: - key_text = str(key) + if isinstance(key, str): + key_text = key + elif isinstance(key, (bool, int, float, complex)): + key_text = str(key) + else: + # Composite/custom labels can invoke hostile ``__repr__`` methods; + # report a structural placeholder rather than serializing them. + return f"{path}[