From d31c1418c0c23994d1444dbd9d356b9a8238fd5e Mon Sep 17 00:00:00 2001 From: commdata2338 <199849969+commdata2338@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:58:37 -0500 Subject: [PATCH] Research: audit PR1824 payload equivalence and add a strict opt-in comparator On the pinned PR1094 final-stage replay corpus (105 files), PR #1824's reusable comparator classifies 39 mutations as mere formatting or relocation: 25 Boolean/integer substitutions in three receipts, 12 input-path alias cases, one duplicate key and one exact decimal change. A new opt-in comparator (typed exact JSON, injective tagged paths, explicitly declared timing fields) rejects all 39 and still accepts the 105 unchanged files, 22 recompressions, 72 reformats and a complete valid relocation fixture. No frozen caller is rewired; no kappa claim. Adds research/strict-replay-payload-comparison/ only. Submitted by @commdata2338 with substantial AI assistance (OpenAI Codex research and implementation; packaged with Claude Code). --- .../README.md | 196 ++++++++ .../RESULT.json | 466 ++++++++++++++++++ .../SOURCES.json | 54 ++ .../TASK.json | 122 +++++ .../TIMINGS.json | 105 ++++ .../VERIFY.json | 37 ++ .../compare.py | 260 ++++++++++ .../test_compare.py | 138 ++++++ .../verify.py | 278 +++++++++++ 9 files changed, 1656 insertions(+) create mode 100644 research/strict-replay-payload-comparison/README.md create mode 100644 research/strict-replay-payload-comparison/RESULT.json create mode 100644 research/strict-replay-payload-comparison/SOURCES.json create mode 100644 research/strict-replay-payload-comparison/TASK.json create mode 100644 research/strict-replay-payload-comparison/TIMINGS.json create mode 100644 research/strict-replay-payload-comparison/VERIFY.json create mode 100644 research/strict-replay-payload-comparison/compare.py create mode 100644 research/strict-replay-payload-comparison/test_compare.py create mode 100644 research/strict-replay-payload-comparison/verify.py diff --git a/research/strict-replay-payload-comparison/README.md b/research/strict-replay-payload-comparison/README.md new file mode 100644 index 0000000000..20dd7ae4f6 --- /dev/null +++ b/research/strict-replay-payload-comparison/README.md @@ -0,0 +1,196 @@ +# Strict comparison of generated replay payloads + +**Scoped audit and opt-in tool; no new mathematics or kappa.** The retained +[PR1824 comparator](../selected-final-stage-portability/compare_payloads.py) +classifies 39 changes to authenticated PR1094 receipts as mere formatting or +relocation. [compare.py](compare.py) rejects all 39 while retaining valid +formatting, gzip recompression, path relocation and explicitly declared timing +changes. The frozen predecessor, selected record and source pins are unchanged. + +For example, replacing `"local_profile_recounted": true` with +`"local_profile_recounted": 1` in the archived `INDEPENDENT-T16.json` returns +`json-format` in the predecessor. Python object equality equates `True` and `1` +before its type-aware `json_diff()` is reached. The strict tool reports a +`proof-relevant-candidate`, leaving mathematical significance to the reviewer. + +## Exact audit domain and result + +The corpus is all **105 files** under `candidate/strict/` in the immutable +PR1094 reviewed replay archive. The archive and comparator are pinned by raw +SHA256 and Git blob in [SOURCES.json](SOURCES.json). The archived review receipt, +assembly and replay receipt are also bound to their retained files. Archive +members are read in memory, without extraction or execution. + +| Mutation family | Cases | Predecessor classification | Strict classification | +| --- | ---: | --- | --- | +| Boolean ↔ integer 0/1 | 25 | `json-format` | proof-relevant candidate | +| Replace an input path by its literal placeholder text | 6 | `json-relocated` | proof-relevant candidate | +| Insert a conflicting placeholder key that relocation overwrites | 6 | `json-relocated` | proof-relevant candidate | +| Add a duplicate proof-field key | 1 | `json-format` | invalid (duplicate key) | +| Change 68 to 68.000000000000000000000001 | 1 | `json-format` | proof-relevant candidate | + +The 25 type cases exhaust the eligible scalar leaves of **three receipts only**: +`INDEPENDENT-T16.json`, `REPLAY.json` and `STAGES.json`. They comprise two Boolean +fields, one namespace coordinate and 22 stage exit codes. This is not an +exhaustive mutation census of the large generated word arrays. The 12 path +cases cover each of the six input keys, with both replacement and insertion. +[RESULT.json](RESULT.json) records every case, its typed path, mutated-byte +digest and both classifications. + +The relocation issue is independent of type equality. The predecessor converts +paths to ordinary placeholder strings and rebuilds objects with a dictionary +comprehension. An existing literal placeholder key can therefore collide with +a relocated path; one value disappears. In the insertion witnesses the extra +key has an incorrect all-zero digest and is silently overwritten. The strict +representation retains the extra key and value. + +Positive controls check all 105 unchanged files, all **22** gzip files under an +alternate compression level/mtime, and all **72** plain JSON files after +reformatting. A separately constructed complete corpus fixture changes only +six root prefixes and 22 valid timing values; its classifications are 102 +byte-identical, one relocated JSON, and two declared-timing files. This fixture +is a transformation of reviewed output, not a new supplier execution. +Deleting each of the 22 declared timing fields is rejected separately. + +## Comparison contract + +The tool compares complete parsed JSON trees, retaining empty containers, +unknown fields, object keys and array positions. Duplicate keys and nonfinite +constants are invalid, even when both files contain identical bytes. Decimal +numbers are parsed exactly, so an arbitrarily small decimal change cannot +disappear through binary floating-point rounding. Booleans, integers and decimal +numbers have distinct tags. Decimal spellings with the same exact value, such +as `1.25` and `125e-2`, may compare equal; integer `1` versus decimal `1.0` is +conservatively reported as a type change. Integer negative zero is integer zero. + +Relocation maps an explicitly declared absolute POSIX prefix to an internal +`(path, token, suffix)` tag. Literal strings have a different tag. Each side +must use distinct tokens and prefixes, and the token sets must agree across +sides. Overlapping prefixes are allowed with distinct tokens and longest-prefix +matching. There is no filesystem resolution or Windows-path interpretation. +Only prefix substitution is permitted; suffixes are retained literally. + +This map is injective within either input: unequal suffixes or tokens stay +unequal, while a fixed token identifies one prefix. A literal string cannot +equal a tagged path. Object canonicalization retains all key/value pairs and +checks that their tagged keys are distinct. By structural induction, the tagged +tree then preserves every node except the explicitly declared quotients. +Identical bytes are accepted only after parsing and timing validation; when root +interpretations differ, their full relocated trees must still agree. + +No timing field is ignored by default. [TIMINGS.json](TIMINGS.json) opts into +the exact 22 typed locations for these two selected receipts. Every declared +location must exist in both files and contain a finite nonnegative number, +excluding Booleans. It then receives a timing sentinel. A missing field, +an unfamiliar nested `seconds`, a key containing `/seconds`, a changed stage +name or a changed digest remains visible. In particular, **raw digest fields +are never ignored or rebound** by this tool. + +For gzip files the complete decompressed content is compared. Non-JSON payloads +require byte equality; binary bytes are never decoded with replacement. Regular-file +inventories must agree and symlinks are outside the contract. Exit code 0 means +equivalence under these explicit rules, 1 means a file difference or invalid +payload, and 2 means a configuration/input-access error. This is comparison, +not mathematical verification or authentication of a claimed execution. + +## Reproduce + +Python 3.12 standard library, from the repository root: + +```sh +python3 -I -B research/strict-replay-payload-comparison/verify.py +python3 -I -O -B research/strict-replay-payload-comparison/test_compare.py +python3 scripts/research_package.py --package research/strict-replay-payload-comparison --check-only +python3 scripts/research_package.py --package research/strict-replay-payload-comparison +``` + +Expected audit marker: + +```text +PASS_STRICT_REPLAY_PAYLOAD_COMPARISON files=105 false_equivalences=39 type_cases=25 path_cases=12 timing_rejections=22 no_new_kappa +``` + +The default audit writes nothing in the checkout and uses no Git or network. +`--write-result` is a first-generation option that refuses to overwrite its +receipt. The registered package checker additionally checks historical Git +provenance. Nine focused test groups cover scalar types and exact decimals, +duplicate/nonfinite JSON, injective roots and stale paths, precise timing paths, +gzip corruption, full containers, CLI failures, and immutable-source rejection. +The archive control changes only gzip MTIME, confirms an unchanged decompressed +tar, and still requires raw-source-pin rejection. + +For other generated trees, use the standalone tool with no exemptions, or add +explicit root pairs and a reviewed timing policy: + +```sh +python3 research/strict-replay-payload-comparison/compare.py REF_DIR RUN_DIR +python3 research/strict-replay-payload-comparison/compare.py REF_DIR RUN_DIR \ + --ref-root 'WORK=/absolute/reference/work' --run-root 'WORK=/absolute/run/work' \ + --timings research/strict-replay-payload-comparison/TIMINGS.json --out comparison.json +``` + +The example root mapping is illustrative. The selected receipt requires both +its `WORK` and `LOCAL` roots, using the paths actually recorded by each run. +Never apply generated-payload equivalence to immutable source archives. + +## Independent Linux observation and limits + +A separate invocation of the unmodified PR1824 `verify.py` on Linux x86_64, +CPython 3.12.3, completed the selected final stage with the retained kappa, but +its expected comparison-count assertion failed. The output had 99 byte-equal +files, one JSON formatting difference, one relocated JSON, two timing differences +and two proof-relevant candidates. The formatting difference was +`global/audit/PREFIX-AND-INVOICE.json`; the two candidates were its raw +`input_sha256/fresh_invoice` references in `ADMISSION-CONTROLS.json` and +`ASSEMBLY.json`. The invoice's complete typed payload agreed. This is additional +local execution evidence, not a registered cross-platform equivalence claim; +the archived byte-exact ASSEMBLY condition did not pass. Neither comparator is +changed to suppress those references. The reproduction is the predecessor's +documented `python3 -B research/selected-final-stage-portability/verify.py`. + +This audit does not establish that the production supplier verifier accepts a +bad construction, invalidate the predecessor's reported macOS runs, rebuild the +earlier cold stages, or discharge inherited all-size, physical, precision, +hosting or analytic obligations. No Windows execution, resource-exhaustion +hardening, formal proof of this Python implementation, new supplier, or exponent +improvement is claimed. The standalone comparator is opt-in; no frozen caller +is rewired. + +## Scope, overlap and credit + +Base: `c676548106012571fefa1dce0ec16f1bcfe11dce`. Card `portable-replay`, priority +revision 3; full refined task card and exact predecessor heads are in +[TASK.json](TASK.json). The claim is in discussion +[#1472](https://github.com/CrocSwap/integer-mult-bounds/discussions/1472). + +Josie Elliston's PR1824 supplied the comparator, replay and portability results +being audited; its exact head is `ad5b1248f25a71983b20617e2f3a80be22f03990`. +Adam Gall's [generated-identity audit](../selected-generated-identity-audit/README.md) +checks a different seven-artifact projection and explicitly leaves other +normalizers open. This account's PR1372 and Gareth Davidson's T37 code concern +digest-to-role permutations, not these type/relocation equivalences. PR1559 +repairs different gzip comparison sites. These contributions are credited, +not repeated. No pending code dependency is imported. + +Credit Maxime Fleury's gzip convention and Antoine Olivier's independent replay; +Rohan Arun's PR1094, Jeremy Kothe's strict verifier, Hao Yan and the retained +supplier/compiler lineage; and Douglas Colkitt's reviewed archive and replay +records. Prepared for commdata2338 (@commdata2338), with substantial OpenAI Codex assistance. +Assistance covered design, code, experiments, tests and writing; self-reviewed +only. New work is Apache-2.0; predecessor bytes and notices remain intact. + +## Open next questions + +1. **Adopt the strict comparator in a future wrapper?** This could remove the + demonstrated ambiguity. First review each caller's intended number/path/time + contract, then test that caller with explicit timing paths. The blocker is + caller-specific compatibility and source-pin review; adoption is not part of + this package. +2. **Bind portable invoice references without ignoring them?** The Linux + observation shows why formatting-independent payload equality alone is not + enough for receipts containing raw hashes. First authenticate both complete + invoice payloads and design an explicit reference-rebinding rule. The blocker + is preserving that binding across the complete receipt graph; a global + `log_sha256` or `input_sha256` exemption would not do so. + +Structured versions of both questions are in [TASK.json](TASK.json). diff --git a/research/strict-replay-payload-comparison/RESULT.json b/research/strict-replay-payload-comparison/RESULT.json new file mode 100644 index 0000000000..9c199b1b5c --- /dev/null +++ b/research/strict-replay-payload-comparison/RESULT.json @@ -0,0 +1,466 @@ +{ + "archive_sha256": "f1da80610bf33c58527f18ad63b4350d0b864e1d4486141ee17ac9bed4d529d6", + "base_commit": "c676548106012571fefa1dce0ec16f1bcfe11dce", + "corpus_files": 105, + "legacy_false_equivalence_counts": { + "boolean-integer": 25, + "duplicate-key": 1, + "exact-decimal-perturbation": 1, + "literal-placeholder-replacement": 6, + "relocation-key-collapse": 6 + }, + "legacy_false_equivalences": [ + { + "candidate_sha256": "2e517a9f98e6d5503760ce87bb5653c6c78ca75877ad658842b4fb71b9a0b227", + "file": "INDEPENDENT-T16.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "local_prefix_from_native_receipt" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "f24b085d20b3e56f05e699094a6f73ce3e74520c4955f5ab85caa286b42102c0", + "file": "INDEPENDENT-T16.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "local_profile_recounted" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "7a5f9a6c6c8148082495c8c1ca798d043743b89930d51865ec5c76d4c97f6dd5", + "file": "INDEPENDENT-T16.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "namespace", + "data", + 0 + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "28cd53285513dbd0728946dfe922685b1545556bd0958291eaa6641512c50f80", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 0, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "41d863d9e5558826ec6c30c8813da343de14bc814f84e5b8cc57280c9496c8ca", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 1, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "621ea9b66e66b4a2ffba717bb88d3162f3c9a681685bab0b1c867cb624ed7407", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 2, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "7ec12e4c501e2c9c8e5cd80141e8dc9967b9562998cf10c37fec119bf105de5c", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 3, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "e4e04921c21d75846bfb973558ed786f8f5b8ff437aca955a146ea98c73d3a44", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 4, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "6812a322d2a18dbe003b8b0c4ee222b41c4ad6b85548115f8e5f3de700627110", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 5, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "b94f46ce1b196c3df9b73802ed30a5e4336eb00eac8ba0e8dce42b12634444ae", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 6, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "ee42966d4fe051e62977543bd64ab84169eac367a268a8551abdaca45e9911d6", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 7, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "d9ef3535dbf62a87f873fb95c148da5765524c84f4eb36d64240fc0b53753076", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 8, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "4e7186101b6a560f43c34e5d03e5913857e7faf3fb3a5764ebbd08d556923416", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 9, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "96bcbd8c9b09ad12ba335de1a4339a5999cc9917ad7de76dc42941e30509820d", + "file": "REPLAY.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + "stages", + 10, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "fcd871a20b538f2ce3480ae1be639f7e32b2e278ea8584fa77a4ecc097127d25", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 0, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "ea8ee509fe4a6886b09622a1a75231baefe0cf82d7db4b73aae88592821ad62a", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 1, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "bce70907e9e80e0bac4736cae1208739f29a38ef862673a47c84ca37a38ab623", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 2, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "1caa51b884eb656a5c62635cc8c7375ae1237b04ae160abf595d3777d210452d", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 3, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "fda114f8beef357fcdbe83767854825d981ffbbf23654cb37bb6a296f588d130", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 4, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "f653bcb3c553fba41951e8a88d5553fcd8ce9c642b472a2544e70c68a189acdd", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 5, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "528b143eec80ae805acca06ce5f97f31febea6fee920904fee04872deab4655a", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 6, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "e06c7148ab061c749abb2868554840794fea8a7c9ef62348fb394339ba0ac196", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 7, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "bec36a95483049a4beba56fd38451d094ae71851323ba790ce8fa64215526e9a", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 8, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "09a5e06aad58c720f9793fdc07e49e3474b61cd64e501f39ebe1d5b5615ad879", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 9, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "7187417feb178b844ba8aeb7e98801dd509b8bac71d3ca6bae69b443b50b1d70", + "file": "STAGES.json", + "kind": "boolean-integer", + "legacy": "json-format", + "path": [ + 10, + "exit_code" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "28861580a76538d34dd16034844398e68cbe3cf63f043e773d3abd3ca0404831", + "file": "INDEPENDENT-T16.json", + "kind": "literal-placeholder-replacement", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1047-selected/slots/local/CANONICAL-WORD.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "c7aea8d1d37b3bd1c19ee6d35fdfa05b3bf1498afbbd8834b8904f5abe6fdc46", + "file": "INDEPENDENT-T16.json", + "kind": "relocation-key-collapse", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1047-selected/slots/local/CANONICAL-WORD.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "aaad035014e40455f6b1ca4f7857343c218b6f7415c1f8b334f518deb4f332f7", + "file": "INDEPENDENT-T16.json", + "kind": "literal-placeholder-replacement", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1047-selected/slots/local/INDEPENDENT-LOCAL-REPLAY.json" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "cd617f46f2fc1ac8ecfe007b28f4b66024820d8605a33621d7960b682a922eb7", + "file": "INDEPENDENT-T16.json", + "kind": "relocation-key-collapse", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1047-selected/slots/local/INDEPENDENT-LOCAL-REPLAY.json" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "31cd733756ea109c87292fdeafb4d8158b926a68e940997d35f24c95f5f62b48", + "file": "INDEPENDENT-T16.json", + "kind": "literal-placeholder-replacement", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/source/data/OWNERS.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "9cfbe2b79ada3df228630d701b6f3be29830f24a25925a5ba6f31015081f36e2", + "file": "INDEPENDENT-T16.json", + "kind": "relocation-key-collapse", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/source/data/OWNERS.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "b11a3adabde8546d8908d64f10474c62773ae41d986e51e43bd85e4d66803c46", + "file": "INDEPENDENT-T16.json", + "kind": "literal-placeholder-replacement", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/strict/global8/base/split-centres/CANONICAL-WORD.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "c58e8538e794abde48aea3bba98e3e136cf1896eaf24efec8545e9ca48ea3931", + "file": "INDEPENDENT-T16.json", + "kind": "relocation-key-collapse", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/strict/global8/base/split-centres/CANONICAL-WORD.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "e0c5617e5b6daca550461506f66a422aeb7923c83a4df95b0fcc191f694196b3", + "file": "INDEPENDENT-T16.json", + "kind": "literal-placeholder-replacement", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/strict/global8/base/split-centres/GLOBAL-TEMPLATE.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "b8f95a7ea59446d7cbe7900118fcf35257f83266c6433cfd77fe985ce0b6564e", + "file": "INDEPENDENT-T16.json", + "kind": "relocation-key-collapse", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/strict/global8/base/split-centres/GLOBAL-TEMPLATE.json.gz" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "1fc2fe191be2bd15196fec907a5dc1364d3571a9e970bbd44fad755103ce19d0", + "file": "INDEPENDENT-T16.json", + "kind": "literal-placeholder-replacement", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/strict/global8/base/work/complex-nine-native/DATA-PHASES.json" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "da06fe2ff95fa870fa07dbf07bfe6750e981ba2a3d4244fe19a00ac05982932e", + "file": "INDEPENDENT-T16.json", + "kind": "relocation-key-collapse", + "legacy": "json-relocated", + "path": [ + "input_sha256", + "/private/tmp/batch1099-selected/candidate/strict/global8/base/work/complex-nine-native/DATA-PHASES.json" + ], + "strict": "json-diff:proof-relevant-candidate" + }, + { + "candidate_sha256": "32f2ae8b651a8cd4b532d935175a344423bdbe21c1b5685712020d0e1a43bd40", + "file": "INDEPENDENT-T16.json", + "kind": "duplicate-key", + "legacy": "json-format", + "path": [ + "local_profile_recounted" + ], + "strict": "invalid" + }, + { + "candidate_sha256": "2d0213477734a610e94600e172705778b1864120c70353dc2c45e8be542b694b", + "file": "INDEPENDENT-T16.json", + "kind": "exact-decimal-perturbation", + "legacy": "json-format", + "path": [ + "local_prefix_bits" + ], + "strict": "json-diff:proof-relevant-candidate" + } + ], + "legacy_sha256": "3aa3459404265188e51a4a74a57f6c01d9ea7cad205edb962e5d51c5a45a50cb", + "missing_timing_fields_rejected": 22, + "positive_controls": { + "recompressed_files": 22, + "reformatted_files": 72, + "unchanged_files": 105 + }, + "relocation_timing_classes": { + "bytes": 102, + "json-diff:declared-timing": 2, + "json-relocated": 1 + }, + "result_kind": "scoped-audit-and-opt-in-tool", + "schema_version": 1, + "scope": "Comparison audit only; no cold replay, supplier admission or new kappa." +} diff --git a/research/strict-replay-payload-comparison/SOURCES.json b/research/strict-replay-payload-comparison/SOURCES.json new file mode 100644 index 0000000000..0c1e7c75b3 --- /dev/null +++ b/research/strict-replay-payload-comparison/SOURCES.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "base_commit": "c676548106012571fefa1dce0ec16f1bcfe11dce", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "ae6a99f28aa468649eb66a361e764d769363bf11", + "sha256": "10b9e340fd8e354a37a0c4813c90e5de1dd42044b5f6aca4a3fc5a42b685fa32" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "06c96f5579bb67ed1b86d334e78fbed7da6945f1", + "sha256": "6686ad58f1fbdbc87134a78901b60fd53a20d3984355e91defbbebc9e9fa8075" + }, + { + "path": "research/community-frontier1094-audit/ASSEMBLY.json", + "usage": "input", + "git_blob": "5fd676df060f3b450782fdf5ff5f6beb63a5eeb0", + "sha256": "1122cb3a575f8c788345fc1bf4c108ac7c3250fb81a5988671f5eb04b3ab3045" + }, + { + "path": "research/community-frontier1094-audit/REPLAY.json", + "usage": "input", + "git_blob": "2cfb7fffa7cd78f418be16c13224c3206b6d0183", + "sha256": "ce0678fa7abaab221de74baeace0c6696d5ea23a36719a58cff230215a06aae7" + }, + { + "path": "research/community-frontier1094-audit/RESULT.json", + "usage": "input", + "git_blob": "131d9c96a54632ae385ced4138fd7f30ad448a3e", + "sha256": "1e3e47e9ec24bcb6fb5f703b7fce0aab658d84ff72bfae5da3ae429d17b6e752" + }, + { + "path": "research/community-frontier1094-audit/selected-replay.tar.gz", + "usage": "input", + "git_blob": "c24daf067f9c543421d89ca2eeaa4eb78a57fb6f", + "sha256": "f1da80610bf33c58527f18ad63b4350d0b864e1d4486141ee17ac9bed4d529d6" + }, + { + "path": "research/selected-final-stage-portability/README.md", + "usage": "input", + "git_blob": "f14313df138b0a6d022e73f5395addd195067306", + "sha256": "13b6ada37c593b172fb73fac58fb615993d8f5c10c7508f9e895c6bb9006e127" + }, + { + "path": "research/selected-final-stage-portability/compare_payloads.py", + "usage": "input", + "git_blob": "a2a2aea73bda06c2358065388a30fd4cbc7d423f", + "sha256": "3aa3459404265188e51a4a74a57f6c01d9ea7cad205edb962e5d51c5a45a50cb" + } + ] +} diff --git a/research/strict-replay-payload-comparison/TASK.json b/research/strict-replay-payload-comparison/TASK.json new file mode 100644 index 0000000000..31e9598cec --- /dev/null +++ b/research/strict-replay-payload-comparison/TASK.json @@ -0,0 +1,122 @@ +{ + "schema_version": 1, + "task_id": "portable-replay", + "priority_revision": 3, + "base_commit": "c676548106012571fefa1dce0ec16f1bcfe11dce", + "original_card": { + "id": "portable-replay", + "question": "Can one reproducible environment or artifact-comparison failure be repaired?", + "starting_points": [ + "research/gzip-container-reproducibility/README.md", + "docs/ci-verification.md" + ], + "methods": [ + "Reproduce one portability failure and add a minimal cross-environment fix.", + "Audit one canonical-payload comparison for omitted proof-relevant fields." + ], + "acceptance_gate": "Preserve immutable source hashes and proof-relevant content; a different compressed byte stream alone is not a mathematical mismatch.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Read the gzip-container reproducibility examples and distinguish frozen source bytes from regenerated payloads.", + "scope": "Positive control / source-bound orientation only; this is not a new construction or full cold replay." + }, + "first_experiment": "Reproduce one specific mismatch or audit one canonical comparison for omitted proof-relevant fields.", + "bottleneck_link": "Portable exact checks prevent environment failures from blocking valid contributions.", + "allowed_variations": [ + "One interpreter/version mismatch or one payload field comparison." + ], + "scope_change_triggers": [ + "Removing immutable-source pins because generated compressed bytes differ." + ], + "overlap_terms": [ + "gzip container", + "reproducibility", + "canonical payload" + ] + }, + "scope_status": "refined", + "actual_scope": "Audit PR1824 selected-final-stage compare_payloads.py for typed-JSON and path-relocation false equivalences on the pinned PR1094 output; provide an opt-in strict successor.", + "scope_change_reason": "The card explicitly permits a canonical-payload omission audit. This fixes the comparator and actual source-bound corpus; it differs from the T37 input-role permutation audit and the generated_identity.py audit.", + "bottleneck_link": "Trustworthy portable payload checks prevent receipt representation differences from concealing changed fields. This is verifier engineering with no circuit, recursive-profile or kappa change.", + "budget_minutes": 60, + "stop_rule": "Stop with exact counterexamples and a checked opt-in comparator; do not generalize to production acceptance or repair unrelated supplier stages. Reserve the final quarter of a 60-minute session for validation and handoff.", + "outcome": "completed", + "evidence": "RESULT.json replays 39 false-equivalence witnesses, all 105 unchanged files, 22 recompressions, 72 JSON reformats, a complete relocation/timing fixture and 22 missing-time rejections. test_compare.py covers independent semantic and integrity controls.", + "author": "commdata2338", + "ai_assistance": "Substantial OpenAI Codex assistance in audit design, implementation, experiments, testing and writing; self-reviewed, no independent human review claimed.", + "open_questions": [ + { + "question": "Should the opt-in strict comparator replace the retained diagnostic comparator in a future replay wrapper?", + "why_it_matters": "The 39 checked counterexamples delimit the reusable tool, while production callers may add separate source and schema checks.", + "first_step": "Review the explicit JSON number/path/time contract and rerun each intended caller with its exact timing-path schema.", + "blocked_by": "No automatic adoption is made here; caller-specific compatibility and source-pin updates need separate review." + }, + { + "question": "Can generated invoice references be rebound to complete payload identities across Python versions?", + "why_it_matters": "A Linux Python 3.12 final-stage run changed invoice JSON formatting and consequently two raw-hash references; globally ignoring those references would drop binding information.", + "first_step": "Reproduce the PR1824 wrapper on Linux, authenticate both complete invoice payloads, then design an explicit source-bound reference-rebinding rule.", + "blocked_by": "This package deliberately preserves raw digest fields; a generic volatile-key exception is insufficient." + } + ], + "chooser": { + "nonce": "993e218fc30808586c2384908063757a", + "track": "verifier-engineering", + "scope_key": "91a9a2afec282c926282", + "refined_scope_key": "PR1824/selected1094-payload-comparator/typed-json-and-injective-relocation" + }, + "predecessors": [ + { + "pr": 1824, + "author": "Josie Elliston (@JosieElliston)", + "head": "ad5b1248f25a71983b20617e2f3a80be22f03990", + "relationship": "Audited comparator and portability baseline; retained on main." + }, + { + "pr": 1094, + "author": "Rohan Arun (@rohanarun)", + "head": "0fc329d0bec93ca7cd288eaea1295172e8789810", + "relationship": "Reviewed final-stage output corpus; no new supplier claim." + }, + { + "pr": 1372, + "author": "commdata2338", + "relationship": "Different T37 digest-role permutation audit, credited and not repeated." + } + ], + "discussion": 1472, + "task_card": { + "object_and_question": "PR1824 classify()/relocate() on the 105 archived PR1094 final-stage files: can type changes or relocation aliases be mistaken for formatting/relocation?", + "exact_claim_audited": "Parsed/relocated JSON equality preserves every field and key. Test the comparator boundary, not selected production acceptance.", + "method_and_domain": "Exact corpus comparison; exhaustive Boolean/integer 0/1 substitutions in INDEPENDENT-T16.json, REPLAY.json and STAGES.json only (25 leaves); replace/insert each of six input keys with its literal placeholder (12 cases); one duplicate-key and one exact-decimal witness; explicit typed timing policy.", + "nearest_overlap": "PR1824 implements the comparator; selected-generated-identity-audit checks a different seven-artifact identity function; PR1372 audits T37 digest-role permutations; PR1559 changes PR589 gzip cmp sites. No pending implementation dependency.", + "cost_and_required_improvement": "No mathematical cost change. Require exact false-equivalence witnesses and rejection by a successor that still accepts valid formatting, recompression, relocation and declared numeric timings.", + "success_and_falsification": "Source pins, full deterministic receipt equality, semantic controls and registered Python 3.12 replay. A valid fixture rejection or a changed undeclared field accepted by the strict comparator falsifies its declared contract.", + "bounded_session": "60 minutes, at most eight busy processes/threads, standard library, no GPU; one bounded pivot maximum.", + "predecessor_credit": "Josie Elliston; Maxime Fleury and Antoine Olivier; Adam Gall; Gareth Davidson; Rohan Arun, Jeremy Kothe, Hao Yan and Douglas Colkitt; retained supplier lineage.", + "artifact_and_reproduction": "python3 -I -B research/strict-replay-payload-comparison/verify.py" + }, + "discussion_url": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1472#discussioncomment-18930555", + "overlap": { + "mode": "offline shared snapshot; complete pagination, expected stale-cache warnings", + "queries": [ + "--task portable-replay", + "--path research/selected-final-stage-portability/compare_payloads.py", + "json-format / relocation collision / duplicate JSON / typed equality", + "selected-final-stage-portability / strict-replay-payload-comparison / relocation-key-collapse / typed JSON" + ], + "open_prs_fetched_at": "2026-10-14T17:42:41.850897+00:00", + "discussions_fetched_at": "2026-10-14T17:44:14.364568+00:00", + "pr_files_fetched_at": "2026-10-14T17:17:53.948388+00:00", + "nearest_threads": [ + 1472, + 1355, + 1884 + ], + "finding": "No competing comparator scope identified; the matching current claim is this investigation. No-match is not a novelty certificate." + } +} diff --git a/research/strict-replay-payload-comparison/TIMINGS.json b/research/strict-replay-payload-comparison/TIMINGS.json new file mode 100644 index 0000000000..648ef4605e --- /dev/null +++ b/research/strict-replay-payload-comparison/TIMINGS.json @@ -0,0 +1,105 @@ +{ + "REPLAY.json": [ + [ + "stages", + 0, + "seconds" + ], + [ + "stages", + 1, + "seconds" + ], + [ + "stages", + 2, + "seconds" + ], + [ + "stages", + 3, + "seconds" + ], + [ + "stages", + 4, + "seconds" + ], + [ + "stages", + 5, + "seconds" + ], + [ + "stages", + 6, + "seconds" + ], + [ + "stages", + 7, + "seconds" + ], + [ + "stages", + 8, + "seconds" + ], + [ + "stages", + 9, + "seconds" + ], + [ + "stages", + 10, + "seconds" + ] + ], + "STAGES.json": [ + [ + 0, + "seconds" + ], + [ + 1, + "seconds" + ], + [ + 2, + "seconds" + ], + [ + 3, + "seconds" + ], + [ + 4, + "seconds" + ], + [ + 5, + "seconds" + ], + [ + 6, + "seconds" + ], + [ + 7, + "seconds" + ], + [ + 8, + "seconds" + ], + [ + 9, + "seconds" + ], + [ + 10, + "seconds" + ] + ] +} diff --git a/research/strict-replay-payload-comparison/VERIFY.json b/research/strict-replay-payload-comparison/VERIFY.json new file mode 100644 index 0000000000..23926c981e --- /dev/null +++ b/research/strict-replay-payload-comparison/VERIFY.json @@ -0,0 +1,37 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "research/community-frontier1094-audit/ASSEMBLY.json", + "research/community-frontier1094-audit/REPLAY.json", + "research/community-frontier1094-audit/RESULT.json", + "research/community-frontier1094-audit/selected-replay.tar.gz", + "research/selected-final-stage-portability/README.md", + "research/selected-final-stage-portability/compare_payloads.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 600, + "checks": "Exact immutable pins and archived receipt binding; complete 105-file positive corpus, 22 recompressions, 72 reformats, source-bound 39-case collision census, exact 22-path timing policy, complete RESULT equality." + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-B", + "{package}/test_compare.py" + ], + "timeout_seconds": 120, + "checks": "Nine semantic control groups including exact numeric types, root injectivity and stale paths, duplicate/nonfinite JSON, explicit typed timings, gzip corruption, complete containers, CLI exit codes and immutable archive-header pin rejection." + } + ] +} diff --git a/research/strict-replay-payload-comparison/compare.py b/research/strict-replay-payload-comparison/compare.py new file mode 100644 index 0000000000..127f143bb8 --- /dev/null +++ b/research/strict-replay-payload-comparison/compare.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Strict, opt-in comparison of generated replay payloads (standard library). + +This does not authenticate a replay or replace immutable source hashes. +Prepared for commdata2338 with substantial OpenAI Codex assistance. Apache-2.0. +""" +import argparse +from decimal import Decimal, InvalidOperation +import gzip +import json +from pathlib import Path, PurePosixPath +import sys +import zlib + + +class InvalidPayload(ValueError): + pass + + +def require(condition, message): + if not condition: + raise InvalidPayload(message) + + +def read_json(raw): + """Reject ambiguous objects; preserve every JSON number exactly.""" + def pairs(items): + result = {} + for key, value in items: + require(key not in result, 'duplicate JSON key: ' + repr(key)) + result[key] = value + return result + + def constant(value): + raise InvalidPayload('nonfinite JSON constant: ' + value) + + return json.loads(raw, object_pairs_hook=pairs, parse_float=Decimal, + parse_constant=constant) + + +class Roots: + """Injective lexical POSIX path map; path tags cannot equal literal text.""" + def __init__(self, pairs=()): + self.pairs = [] + tokens, prefixes = set(), set() + for token, prefix in pairs: + require(type(token) is str and bool(token), 'empty root token') + require(type(prefix) is str and prefix.startswith('/'), + 'root must be an absolute POSIX path') + require(str(PurePosixPath(prefix)) == prefix and + not prefix.startswith('//') and '..' not in prefix.split('/'), + 'root must be lexically canonical') + require(token not in tokens, 'duplicate root token: ' + token) + require(prefix not in prefixes, 'duplicate root prefix: ' + prefix) + tokens.add(token) + prefixes.add(prefix) + self.pairs.append((token, prefix)) + self.tokens = frozenset(tokens) + self.pairs.sort(key=lambda pair: -len(pair[1])) + + def string(self, value): + for token, prefix in self.pairs: + if value == prefix: + return ('path', token, '') + if prefix == '/' and value.startswith('/'): + return ('path', token, value) + if value.startswith(prefix + '/'): + return ('path', token, value[len(prefix):]) + return ('string', value) + + +EMPTY_ROOTS = Roots() + + +def validate_timings(value, paths): + require(type(paths) in (list, tuple), 'timing paths must be an array') + seen = set() + for path in paths: + require(type(path) in (tuple, list), 'timing path must be an array') + require(all(type(p) in (str, int) for p in path), 'invalid timing path component') + path = tuple(path) + require(path not in seen, 'duplicate timing path') + seen.add(path) + node = value + for component in path: + if type(node) is dict and type(component) is str: + require(component in node, 'missing timing field: ' + repr(path)) + elif type(node) is list and type(component) is int: + require(0 <= component < len(node), 'missing timing index: ' + repr(path)) + else: + raise InvalidPayload('timing path has wrong shape: ' + repr(path)) + node = node[component] + require(type(node) in (int, Decimal), 'timing must be numeric, not Boolean/string') + require(node >= 0 and (type(node) is int or node.is_finite()), + 'timing must be finite and nonnegative') + return frozenset(seen) + + +def tree(value, roots=EMPTY_ROOTS, timings=frozenset(), path=()): + """A tagged, lossless tree, except for explicitly declared path/time quotients.""" + if path in timings: + return ('timing',) + if type(value) is dict: + items = [(roots.string(k), tree(v, roots, timings, path + (k,))) + for k, v in value.items()] + require(len({k for k, _ in items}) == len(items), 'relocation key collision') + return ('object', tuple(sorted(items))) + if type(value) is list: + return ('array', tuple(tree(v, roots, timings, path + (i,)) + for i, v in enumerate(value))) + if type(value) is str: + return roots.string(value) + if type(value) is bool: + return ('boolean', value) + if value is None: + return ('null',) + if type(value) is int: + return ('integer', value) + if type(value) is Decimal: + require(value.is_finite(), 'nonfinite decimal') + return ('decimal', value) + raise InvalidPayload('unsupported JSON type: ' + type(value).__name__) + + +def first_difference(a, b, path=()): + """Diagnostic typed path; equality is checked on the entire tagged trees.""" + if a == b: + return None + if a[0] == b[0] == 'object': + left, right = dict(a[1]), dict(b[1]) + for key in sorted(left.keys() | right.keys()): + if key not in left or key not in right: + return {'path': [*path, list(key)], 'kind': 'missing-or-extra-key'} + diff = first_difference(left[key], right[key], (*path, list(key))) + if diff: + return diff + elif a[0] == b[0] == 'array': + if len(a[1]) != len(b[1]): + return {'path': list(path), 'kind': 'array-length'} + for i, (x, y) in enumerate(zip(a[1], b[1])): + diff = first_difference(x, y, (*path, i)) + if diff: + return diff + return {'path': list(path), 'kind': 'value-or-type', + 'reference': repr(a)[:160], 'run': repr(b)[:160]} + + +def classify(name, reference, run, ref_roots=EMPTY_ROOTS, run_roots=EMPTY_ROOTS, + timing_paths=()): + require(ref_roots.tokens == run_roots.tokens, 'root-token sets must agree') + compressed = name.endswith('.gz') + json_name = name[:-3] if compressed else name + try: + a = gzip.decompress(reference) if compressed else reference + b = gzip.decompress(run) if compressed else run + if not json_name.endswith('.json'): + require(not timing_paths, 'timing policy on a non-JSON file') + if reference == run: + return {'cls': 'bytes'} + return {'cls': 'gzip-container' if a == b else 'binary-diff'} + ja, jb = read_json(a), read_json(b) + # Validate even byte-identical inputs: identical malformed JSON is not + # positive evidence, nor is a missing/invalid declared time field. + ta, tb = validate_timings(ja, timing_paths), validate_timings(jb, timing_paths) + # Equal payload bytes with the same path interpretation need no tree + # allocation. Parsing and time validation above still run in full. + if a == b and ref_roots.pairs == run_roots.pairs: + return {'cls': 'bytes' if reference == run else 'gzip-container'} + relocated_a, relocated_b = tree(ja, ref_roots), tree(jb, run_roots) + if relocated_a == relocated_b: + if reference == run: + return {'cls': 'bytes'} + if a == b: + return {'cls': 'gzip-container'} + if not ref_roots.pairs and not run_roots.pairs: + return {'cls': 'json-format'} + if tree(ja) == tree(jb): + return {'cls': 'json-format'} + return {'cls': 'json-relocated'} + timed_a = tree(ja, ref_roots, ta) if ta else relocated_a + timed_b = tree(jb, run_roots, tb) if tb else relocated_b + if timed_a == timed_b: + return {'cls': 'json-diff', 'verdict': 'declared-timing'} + return {'cls': 'json-diff', 'verdict': 'proof-relevant-candidate', + 'first_difference': first_difference(timed_a, timed_b)} + except (ValueError, UnicodeError, OSError, EOFError, zlib.error, InvalidOperation) as exc: + return {'cls': 'invalid', 'reason': str(exc)} + + +def accepted(result): + return result['cls'] in {'bytes', 'gzip-container', 'json-format', 'json-relocated'} or ( + result['cls'] == 'json-diff' and result['verdict'] == 'declared-timing') + + +def inventory(root): + require(root.is_dir(), 'comparison root is not a directory: ' + str(root)) + require(not root.is_symlink(), 'comparison root must not be a symlink') + result = {} + for path in sorted(root.rglob('*')): + require(not path.is_symlink(), 'symlink outside the file-comparison contract') + if path.is_file(): + result[path.relative_to(root).as_posix()] = path + return result + + +def compare(reference, run, ref_roots=EMPTY_ROOTS, run_roots=EMPTY_ROOTS, timings=None): + require(ref_roots.tokens == run_roots.tokens, 'root-token sets must agree') + timings = timings or {} + left, right = inventory(reference), inventory(run) + require(set(timings) <= left.keys() & right.keys(), 'timing policy names absent files') + result = {} + for name in sorted(left.keys() | right.keys()): + if name not in right: + result[name] = {'cls': 'missing'} + elif name not in left: + result[name] = {'cls': 'extra'} + else: + result[name] = classify(name, left[name].read_bytes(), right[name].read_bytes(), + ref_roots, run_roots, timings.get(name, ())) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('reference', type=Path) + parser.add_argument('run', type=Path) + parser.add_argument('--ref-root', action='append', default=[], metavar='TOKEN=/absolute/path') + parser.add_argument('--run-root', action='append', default=[], metavar='TOKEN=/absolute/path') + parser.add_argument('--timings', type=Path, help='JSON object: filename -> exact typed path arrays') + parser.add_argument('--out', type=Path, help='write detailed JSON to a new file') + args = parser.parse_args() + try: + def roots(values): + pairs = [] + for value in values: + token, sep, prefix = value.partition('=') + require(bool(sep), 'expected TOKEN=/absolute/path') + pairs.append((token, prefix)) + return Roots(pairs) + policy = read_json(args.timings.read_bytes()) if args.timings else {} + require(type(policy) is dict, 'timing policy must be an object') + result = compare(args.reference, args.run, roots(args.ref_root), roots(args.run_root), policy) + counts = {} + for value in result.values(): + key = value['cls'] + (':' + value['verdict'] if 'verdict' in value else '') + counts[key] = counts.get(key, 0) + 1 + report = {'equivalent': all(map(accepted, result.values())), 'counts': counts, 'files': result} + if args.out: + with args.out.open('x', encoding='utf-8') as stream: + json.dump(report, stream, indent=2, sort_keys=True) + stream.write('\n') + print(json.dumps({'equivalent': report['equivalent'], 'counts': counts}, sort_keys=True)) + return 0 if report['equivalent'] else 1 + except (ValueError, OSError) as exc: + print('INVALID_COMPARISON: ' + str(exc), file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/research/strict-replay-payload-comparison/test_compare.py b/research/strict-replay-payload-comparison/test_compare.py new file mode 100644 index 0000000000..16c1e94345 --- /dev/null +++ b/research/strict-replay-payload-comparison/test_compare.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Semantic and failure controls for the opt-in comparator. Apache-2.0. + +Prepared for commdata2338 with substantial OpenAI Codex assistance. +""" +import gzip +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import compare as c +import verify as audit + + +class ComparisonControls(unittest.TestCase): + def classify(self, a, b, **kwargs): + return c.classify('receipt.json', a, b, **kwargs) + + def test_exact_typed_scalars_at_every_container_shape(self): + pairs = [(b'true', b'1'), (b'false', b'0'), (b'1', b'1.0'), + (b'9007199254740992.0', b'9007199254740993.0'), + (b'68', b'68.000000000000000000000001')] + for left, right in pairs: + for before, after in ((b'', b''), (b'[', b']'), (b'{"x":', b'}')): + with self.subTest(left=left, before=before): + self.assertFalse(c.accepted(self.classify(before + left + after, + before + right + after))) + self.assertEqual(self.classify(b'{"n":1.2500}', b'{"n":125e-2}')['cls'], 'json-format') + + def test_ambiguous_and_nonfinite_inputs_even_when_identical(self): + for raw in (b'{"rank":0,"rank":1}', b'NaN', b'Infinity', b'-Infinity', b'{'): + with self.subTest(raw=raw): + self.assertEqual(self.classify(raw, raw)['cls'], 'invalid') + # Arbitrarily large *finite* decimal exponents are not converted to float infinity. + self.assertEqual(self.classify(b'1e999', b'10e998')['cls'], 'json-format') + + def test_root_mapping_is_injective_and_keeps_literal_tokens(self): + roots = c.Roots([('W', '/work'), ('N', '/work/nested')]) + values = ['/work', '/work/a', '/work/nested', '/work/nested/a', '/workish/a', + 'W/a', 'N/a', '/a', '/outside/a', '', '/work/a/../b'] + self.assertEqual(len({roots.string(v) for v in values}), len(values)) + left, right = c.Roots([('W', '/a')]), c.Roots([('W', '/b')]) + self.assertTrue(c.accepted(self.classify(b'{"/a/file":1}', b'{"/b/file":1}', + ref_roots=left, run_roots=right))) + for raw in (b'{"W/file":1}', b'{"/b/file":1,"W/file":0}', + b'{"/a/file":1}', b'{"/b/file2":1}'): + self.assertFalse(c.accepted(self.classify(b'{"/a/file":1}', raw, + ref_roots=left, run_roots=right))) + # The identical-byte shortcut cannot hide a stale absolute input path. + self.assertFalse(c.accepted(self.classify(b'"/a/file"', b'"/a/file"', + ref_roots=left, run_roots=right))) + + def test_invalid_root_declarations(self): + for pairs in ([('W', '/a'), ('W', '/b')], [('A', '/a'), ('B', '/a')], + [('W', 'relative')], [('W', '/a/../b')], [('W', '/a/')]): + with self.subTest(pairs=pairs), self.assertRaises(c.InvalidPayload): + c.Roots(pairs) + with self.assertRaises(c.InvalidPayload): + self.classify(b'{}', b'{}', ref_roots=c.Roots([('A', '/a')]), + run_roots=c.Roots([('B', '/b')])) + + def test_timing_is_an_exact_typed_path(self): + policy = [('stages', 0, 'seconds')] + left = b'{"stages":[{"seconds":1.0}],"proof/seconds":17}' + good = b'{"stages":[{"seconds":2}],"proof/seconds":17}' + self.assertTrue(c.accepted(self.classify(left, good, timing_paths=policy))) + changed = b'{"stages":[{"seconds":2}],"proof/seconds":18}' + self.assertFalse(c.accepted(self.classify(left, changed, timing_paths=policy))) + for value in (b'true', b'null', b'-1', b'"2"', b'NaN'): + raw = b'{"stages":[{"seconds":' + value + b'}],"proof/seconds":17}' + self.assertEqual(self.classify(left, raw, timing_paths=policy)['cls'], 'invalid') + for path in ([('stages', False, 'seconds')], [('stages', -1, 'seconds')], + [('stages', '0', 'seconds')], [policy[0], policy[0]], None): + self.assertEqual(self.classify(left, good, timing_paths=path)['cls'], 'invalid') + + def test_gzip_covers_complete_payload_and_corruption(self): + left = gzip.compress(b'{"rank":1}', mtime=0) + right = gzip.compress(b'{"rank":1}', mtime=1) + self.assertEqual(c.classify('word.json.gz', left, right)['cls'], 'gzip-container') + for raw in (gzip.compress(b'{"rank":true}'), gzip.compress(b'{"rank":2}'), left[:-5]): + self.assertFalse(c.accepted(c.classify('word.json.gz', left, raw))) + ambiguous = gzip.compress(b'{"rank":1,"rank":2}') + self.assertEqual(c.classify('word.json.gz', ambiguous, ambiguous)['cls'], 'invalid') + + def test_complete_containers_and_unfamiliar_fields(self): + for a, b in ((b'{}', b'{"new":{}}'), (b'[]', b'[null]'), + (b'{"a":[]}', b'{"a":{}}'), (b'{"x/y":1}', b'{"x":{"y":1}}'), + (b'{"proof":{"seconds":1}}', b'{"proof":{"seconds":2}}')): + self.assertFalse(c.accepted(self.classify(a, b))) + self.assertEqual(c.classify('raw.bin', b'\xff', b'\xfe')['cls'], 'binary-diff') + + def test_cli_exit_status_and_inventory(self): + with tempfile.TemporaryDirectory(prefix='strict-cli-') as temporary: + root = Path(temporary) + left, right = root / 'left', root / 'right' + left.mkdir(); right.mkdir() + a, b = left / 'a.json', right / 'a.json' + a.write_bytes(b'{"checked":true}'); b.write_bytes(b'{ "checked": true }') + command = [sys.executable, '-I', '-O', '-B', str(HERE / 'compare.py'), str(left), str(right)] + def run(): + return subprocess.run(command, capture_output=True, text=True, check=False) + result = run() + self.assertEqual(result.returncode, 0, result.stderr) + b.write_bytes(b'{"checked":1}') + self.assertEqual(run().returncode, 1) + b.unlink() + self.assertEqual(c.compare(left, right)['a.json']['cls'], 'missing') + b.write_bytes(a.read_bytes()); (right / 'extra').write_bytes(b'x') + self.assertEqual(c.compare(left, right)['extra']['cls'], 'extra') + self.assertEqual(run().returncode, 1) + command += ['--ref-root', 'X=/a'] + self.assertEqual(run().returncode, 2) + + def test_source_pin_is_not_a_payload_quotient(self): + with tempfile.TemporaryDirectory(prefix='strict-source-') as temporary: + root = Path(temporary) + for name in audit.INPUTS: + target = root / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(audit.ROOT / name, target) + audit.bound_inputs(root) + path = root / audit.ARCHIVE + raw = bytearray(path.read_bytes()) + raw[4] ^= 1 # only the gzip MTIME header; decompressed tar is unchanged + self.assertEqual(gzip.decompress(raw), gzip.decompress(path.read_bytes())) + path.write_bytes(raw) + with self.assertRaisesRegex(ValueError, 'source SHA256 mismatch'): + audit.bound_inputs(root) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/research/strict-replay-payload-comparison/verify.py b/research/strict-replay-payload-comparison/verify.py new file mode 100644 index 0000000000..8e9f9d7682 --- /dev/null +++ b/research/strict-replay-payload-comparison/verify.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Source-bound comparison audit; no supplier execution or network. + +Prepared for commdata2338 with substantial OpenAI Codex assistance. Apache-2.0. +""" +import argparse +from collections import Counter +import copy +import gzip +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import sys +import tarfile +import tempfile + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(HERE)) +import compare as strict + +ARCHIVE = 'research/community-frontier1094-audit/selected-replay.tar.gz' +LEGACY = 'research/selected-final-stage-portability/compare_payloads.py' +INPUTS = frozenset((ARCHIVE, LEGACY, + 'research/selected-final-stage-portability/README.md', + 'research/community-frontier1094-audit/RESULT.json', + 'research/community-frontier1094-audit/REPLAY.json', + 'research/community-frontier1094-audit/ASSEMBLY.json')) +REF_PAIRS = (('', '/private/tmp/batch1099-selected/candidate'), + ('', '/private/tmp/batch1047-selected/slots/local')) +RUN_PAIRS = (('', '/relocated/replay'), ('', '/relocated/local')) +RECEIPTS = ('INDEPENDENT-T16.json', 'REPLAY.json', 'STAGES.json') + + +def check(condition, message): + if not condition: + raise ValueError(message) + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def encoded(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def bound_inputs(root=ROOT): + manifest = strict.read_json((HERE / 'SOURCES.json').read_bytes()) + rows = [r for r in manifest['files'] if r['usage'] == 'input'] + check(len(rows) == len(INPUTS) and {r['path'] for r in rows} == INPUTS, + 'source input inventory differs') + result = {} + for row in rows: + raw = (root / row['path']).read_bytes() + check(sha(raw) == row['sha256'], 'source SHA256 mismatch: ' + row['path']) + blob = hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() + check(blob == row['git_blob'], 'source Git blob mismatch: ' + row['path']) + result[row['path']] = raw + return result + + +def corpus(inputs): + result = {} + with tarfile.open(fileobj=io.BytesIO(inputs[ARCHIVE])) as archive: + names = [m.name for m in archive.getmembers()] + check(len(names) == len(set(names)), 'duplicate archive member') + prefix = 'candidate/strict/' + for member in archive.getmembers(): + if not member.name.startswith(prefix) or member.isdir(): + continue + check(member.isfile(), 'nonregular corpus member') + name = member.name[len(prefix):] + check(name and not name.startswith('/') and '..' not in name.split('/'), + 'invalid corpus member name') + result[name] = archive.extractfile(member).read() + check(archive.extractfile('RESULT.json').read() == + inputs['research/community-frontier1094-audit/RESULT.json'], + 'archived review receipt differs') + check(len(result) == 105, 'reviewed corpus must contain 105 files') + for name in ('ASSEMBLY.json', 'REPLAY.json'): + check(result[name] == inputs['research/community-frontier1094-audit/' + name], + 'archived ' + name + ' differs') + receipt = strict.read_json(inputs['research/community-frontier1094-audit/RESULT.json']) + check(sha(result['ASSEMBLY.json']) == receipt['assembly_sha256'], 'assembly receipt binding') + return result + + +def legacy_module(): + spec = importlib.util.spec_from_file_location('pinned_pr1824_comparator', ROOT / LEGACY) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def leaves(node, path=()): + if type(node) is dict: + for key, value in sorted(node.items()): + yield from leaves(value, path + (key,)) + elif type(node) is list: + for i, value in enumerate(node): + yield from leaves(value, path + (i,)) + else: + yield path, node + + +def replace(node, path, value): + result = copy.deepcopy(node) + parent = result + for component in path[:-1]: + parent = parent[component] + parent[path[-1]] = value + return result + + +def label(result): + return result['cls'] + (':' + result['verdict'] if 'verdict' in result else '') + + +def audit(inputs, data): + legacy = legacy_module() # only after exact source admission + ref_roots, run_roots = strict.Roots(REF_PAIRS), strict.Roots(RUN_PAIRS) + expected_policy = {'REPLAY.json': [['stages', i, 'seconds'] for i in range(11)], + 'STAGES.json': [[i, 'seconds'] for i in range(11)]} + policy = strict.read_json((HERE / 'TIMINGS.json').read_bytes()) + check(encoded(policy) == encoded(expected_policy), 'timing policy differs from exact 22 paths') + positive = Counter() + for name, raw in sorted(data.items()): + result = strict.classify(name, raw, raw, timing_paths=policy.get(name, ())) + check(label(result) == 'bytes', 'unchanged corpus rejected: ' + name) + positive['unchanged_files'] += 1 + if name.endswith('.gz'): + altered = gzip.compress(gzip.decompress(raw), compresslevel=1, mtime=12345) + result = strict.classify(name, raw, altered) + check(label(result) == 'gzip-container', 'valid recompression rejected: ' + name) + positive['recompressed_files'] += 1 + elif name.endswith('.json'): + # Independent fixture producer; acceptance still uses exact Decimal + # parsing and complete trees, not this ordinary parser's equality. + altered = json.dumps(json.loads(raw), sort_keys=True, indent=3).encode() + result = strict.classify(name, raw, altered, timing_paths=policy.get(name, ())) + check(label(result) == 'json-format', 'valid reformat rejected: ' + name) + positive['reformatted_files'] += 1 + + # Construct a complete relocation/time fixture without executing a supplier. + relocated = dict(data) + value = json.loads(data['INDEPENDENT-T16.json']) + relocated_keys = {} + for key, digest in value['input_sha256'].items(): + matches = [(old, new) for (_, old), (_, new) in zip(REF_PAIRS, RUN_PAIRS) + if key.startswith(old + '/')] + check(len(matches) == 1, 'fixture path must match exactly one source root') + old, new = matches[0] + relocated_keys[new + key[len(old):]] = digest + check(len(relocated_keys) == 6, 'six distinct input roles required') + value['input_sha256'] = relocated_keys + relocated['INDEPENDENT-T16.json'] = encoded(value) + for name in ('REPLAY.json', 'STAGES.json'): + value = json.loads(data[name]) + stages = value['stages'] if type(value) is dict else value + for i, stage in enumerate(stages): + stage['seconds'] = 1000 + i + relocated[name] = encoded(value) + classes = Counter(label(strict.classify(name, raw, relocated[name], ref_roots, run_roots, + policy.get(name, ()))) + for name, raw in sorted(data.items())) + check(dict(classes) == {'bytes': 102, 'json-relocated': 1, 'json-diff:declared-timing': 2}, + 'complete relocation/timing fixture differs') + + witnesses = [] + with tempfile.TemporaryDirectory(prefix='strict-payload-audit-') as temporary: + reference, candidate = Path(temporary) / 'reference.json', Path(temporary) / 'candidate.json' + + def witness(kind, name, changed, path, old_expected, roots=False): + reference.write_bytes(data[name]) + candidate.write_bytes(changed) + mappings = REF_PAIRS if roots else () + old = legacy.classify(reference, candidate, mappings, mappings) + new = strict.classify(name, data[name], changed, + ref_roots if roots else strict.EMPTY_ROOTS, + ref_roots if roots else strict.EMPTY_ROOTS, + policy.get(name, ())) + check(label(old) == old_expected, 'legacy witness not reproduced: ' + kind) + check(not strict.accepted(new), 'strict comparison missed: ' + kind) + witnesses.append({'kind': kind, 'file': name, 'path': list(path), + 'candidate_sha256': sha(changed), 'legacy': label(old), + 'strict': label(new)}) + + # Exhaustive for Boolean and integer 0/1 leaves of these three receipts, + # not for the large generated word arrays in the other corpus files. + for name in RECEIPTS: + original = json.loads(data[name]) + for path, value in leaves(original): + if type(value) is bool: + changed = int(value) + elif type(value) is int and value in (0, 1): + changed = bool(value) + else: + continue + witness('boolean-integer', name, encoded(replace(original, path, changed)), + path, 'json-format') + + name = 'INDEPENDENT-T16.json' + original = json.loads(data[name]) + for key, digest in sorted(original['input_sha256'].items()): + tokenized = legacy.relocate(key, REF_PAIRS) + check(tokenized != key, 'fixture input is outside the known roots') + changed = copy.deepcopy(original) + del changed['input_sha256'][key] + changed['input_sha256'][tokenized] = digest + witness('literal-placeholder-replacement', name, encoded(changed), + ('input_sha256', key), 'json-relocated', roots=True) + changed = copy.deepcopy(original) + changed['input_sha256'] = {tokenized: '0' * 64, **original['input_sha256']} + # Preserve order so the path overwrites the conflicting literal key. + raw = json.dumps(changed, separators=(',', ':')).encode() + witness('relocation-key-collapse', name, raw, ('input_sha256', key), + 'json-relocated', roots=True) + + raw = data[name] + needle = b'"local_profile_recounted": true' + check(raw.count(needle) == 1, 'duplicate-key fixture source shape') + changed = raw.replace(needle, b'"local_profile_recounted": false, ' + needle) + witness('duplicate-key', name, changed, ('local_profile_recounted',), 'json-format') + needle = b'"local_prefix_bits": 68' + check(raw.count(needle) == 1, 'decimal fixture source shape') + changed = raw.replace(needle, b'"local_prefix_bits": 68.000000000000000000000001') + witness('exact-decimal-perturbation', name, changed, ('local_prefix_bits',), 'json-format') + + timing_rejections = 0 + for name, paths in policy.items(): + original = json.loads(data[name]) + for path in paths: + changed = copy.deepcopy(original) + parent = changed + for component in path[:-1]: + parent = parent[component] + del parent[path[-1]] + result = strict.classify(name, data[name], encoded(changed), timing_paths=paths) + check(result['cls'] == 'invalid', 'missing declared timing was accepted') + timing_rejections += 1 + + kinds = Counter(w['kind'] for w in witnesses) + check(dict(kinds) == {'boolean-integer': 25, 'literal-placeholder-replacement': 6, + 'relocation-key-collapse': 6, 'duplicate-key': 1, + 'exact-decimal-perturbation': 1}, 'mutation coverage changed') + return {'schema_version': 1, 'result_kind': 'scoped-audit-and-opt-in-tool', + 'base_commit': 'c676548106012571fefa1dce0ec16f1bcfe11dce', + 'archive_sha256': sha(inputs[ARCHIVE]), 'legacy_sha256': sha(inputs[LEGACY]), + 'corpus_files': len(data), 'positive_controls': dict(sorted(positive.items())), + 'relocation_timing_classes': dict(sorted(classes.items())), + 'legacy_false_equivalence_counts': dict(sorted(kinds.items())), + 'legacy_false_equivalences': witnesses, + 'missing_timing_fields_rejected': timing_rejections, + 'scope': 'Comparison audit only; no cold replay, supplier admission or new kappa.'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--write-result', action='store_true', help='create RESULT.json exclusively') + args = parser.parse_args() + inputs = bound_inputs() + result = audit(inputs, corpus(inputs)) + if args.write_result: + with (HERE / 'RESULT.json').open('x', encoding='utf-8') as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write('\n') + else: + stored = strict.read_json((HERE / 'RESULT.json').read_bytes()) + check(encoded(result) == encoded(stored), 'complete audit receipt differs') + print('PASS_STRICT_REPLAY_PAYLOAD_COMPARISON files=105 false_equivalences=39 ' + 'type_cases=25 path_cases=12 timing_rejections=22 no_new_kappa') + + +if __name__ == '__main__': + main()