From 8c62e5f52f1d172bb8c95e795449205015bb5143 Mon Sep 17 00:00:00 2001 From: Sagar Thakrar <18622288+thakrarsagar@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:04:43 +0530 Subject: [PATCH] Add source-bound JSON reuse for T16 boundary replay --- research/t16-invoice-read-cache/README.md | 123 ++++++++++++ research/t16-invoice-read-cache/RESULT.json | 158 +++++++++++++++ research/t16-invoice-read-cache/SOURCES.json | 102 ++++++++++ research/t16-invoice-read-cache/TASK.json | 95 +++++++++ research/t16-invoice-read-cache/VERIFY.json | 42 ++++ .../t16-invoice-read-cache/test_verify.py | 184 ++++++++++++++++++ research/t16-invoice-read-cache/verify.py | 102 ++++++++++ 7 files changed, 806 insertions(+) create mode 100644 research/t16-invoice-read-cache/README.md create mode 100644 research/t16-invoice-read-cache/RESULT.json create mode 100644 research/t16-invoice-read-cache/SOURCES.json create mode 100644 research/t16-invoice-read-cache/TASK.json create mode 100644 research/t16-invoice-read-cache/VERIFY.json create mode 100644 research/t16-invoice-read-cache/test_verify.py create mode 100644 research/t16-invoice-read-cache/verify.py diff --git a/research/t16-invoice-read-cache/README.md b/research/t16-invoice-read-cache/README.md new file mode 100644 index 0000000000..62df0b4d10 --- /dev/null +++ b/research/t16-invoice-read-cache/README.md @@ -0,0 +1,123 @@ +# Reuse JSON parses in the complete T16 invoice boundary replay + +This opt-in adapter runs Tairitsu-Aya's complete 54-case +[T16 invoice boundary audit](../t16-invoice-boundary-audit/README.md) while +reusing the loaded invoice's JSON parses. Every predicate in that boundary +audit still runs. The complete reconstructed receipt must equal the unchanged +`EXPECTED.json`. No selected source, frozen archive or kappa changes. + +Kumagi's [PR3009](https://github.com/CrocSwap/integer-mult-bounds/pull/3009) +already reduced fixture compression costs. Its remaining handoff identified +repeated JSON parsing. Profiling the current predecessor on this Linux host +with CPython 3.14 found 1,968 `json.raw_decode` calls taking 28.846 seconds of +a 166.859-second instrumented run. These are profile costs, not speedup data. + +## Measured result + +Four fresh CPython 3.12.15 processes on one shared Linux host ran in +baseline/cached/cached/baseline order. Every complete emitted receipt was +byte-identical to the original. [RESULT.json](RESULT.json) records exact commands, +environment, verifier hash, timing method and all observations. + +| Mode | Wall seconds, in execution order | Peak RSS (KiB) | +| --- | --- | --- | +| Baseline | 179.276, 197.756 | 939892, 939804 | +| Cached | 170.213, 179.694 | 970392, 970320 | + +Mean wall time fell from 188.516 to 174.954 seconds (7.19%); mean peak RSS rose +by 30,508 KiB (about 29.8 MiB). Each cached replay reused 488 of 628 reads across +12 paths, retaining 30,013,304 bytes of raw and serialized data. Timing ranges +overlap and the host was shared: these are local observations, not a confidence +interval or portable speed guarantee. No process reused another process's cache. + +## Equivalence and invalidation + +Only the loaded `invoice_allcold.read` function is replaced. Its frozen source +bytes remain unchanged. The adapter keeps one `(raw bytes, marshaled JSON)` +entry for the first successfully parsed contents of each path: + +1. Read the current file bytes on **every** call. Size, modification time, + inode and a previously recorded digest are never freshness tests. +2. If the bytes differ, use the original `json.loads`/`gzip.decompress` + semantics. One-shot mutations are not serialized or added to the cache. +3. If the bytes agree, deserialize the private in-memory representation. + Every call returns a fresh mutable value, including nested lists/dictionaries. + +A hit therefore returns the same JSON value as a fresh parse of the bytes just +read; a miss performs that parse. Mutating the return cannot change the cached +bytes. Malformed inputs and missing files still fail after a successful read. +Restoring a fixture is observed like any other change. `marshal` is used only +on this process's JSON values, never on input-controlled serialized bytes. +The predecessor already uses this representation for its mutation copies. +Keeping the first contents suits this audit's restored-fixture pattern. A file +that changes permanently will be reparsed on every subsequent read; this is +not intended as a general-purpose workload cache. + +There is no persistent cache and no cached assertion or invoice result. +Each replay starts with an empty cache and +removes its extracted fixtures and private bytecode namespace on completion. + +Source pins cover predecessor code, cases, expected receipt, manifests and all +nine transitive data/engine inputs. Archive-member checks, file hashes, +real-file mutations, the JSON assembly boundary, upstream controller checks, +the isolated driver predicate and final restored-fixture check remain active. +Predecessor harness sources are compiled directly; frozen imports use a fresh +bytecode namespace even for direct invocation. The replay refuses `-O`. + +## Reproduce + +From the repository root, standard-library Python 3.12: + +```sh +python3.12 -B research/t16-invoice-read-cache/test_verify.py +python3.12 -B scripts/research_package.py --package research/t16-invoice-read-cache +``` + +Compare the same fixture with and without the adapter, using disposable paths: + +```sh +python3.12 -B research/t16-invoice-read-cache/verify.py --baseline --receipt /tmp/t16-before.json +python3.12 -B research/t16-invoice-read-cache/verify.py --receipt /tmp/t16-after.json +cmp /tmp/t16-before.json /tmp/t16-after.json +cmp /tmp/t16-after.json research/t16-invoice-boundary-audit/EXPECTED.json +``` + +Both modes execute the same authenticated predecessor `run()` and compare its +complete result to the original receipt. Baseline mode leaves its reader +unchanged. Cases still give 28 invoice rejections and 26 acceptances; assembly +rejects 16 survivors and accepts ten, with the original further classifications. +The extra final cache-statistics line is not part of the receipt. + +Ten focused controls cover plain/gzip types, byte changes with preserved timestamps, +repeated changed contents and restoration, cold/warm return mutation, +malformed/truncated data, missing paths, path/suffix separation, source tampering +and rejection of optimized replay. Local mutation experiments confirm that metadata-only +freshness and shared mutable returns fail these controls. The cache tests do +not replace any check of the predecessor's mathematical hypotheses. + +## Scope and sources + +Base: `463113d675af7410ac91d53a4fed51ec6494eb9a`, priority revision 6. +The [canonical claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/2051#discussioncomment-19001801) +continues the original audit's thread. `TASK.json` records scope and overlap. +This is verifier engineering. It neither runs the full selected cold chain nor +discharges any all-size supplier or multiplication theorem obligation. + +## Open next questions + +Does the improvement persist on another Python 3.12 host? Run the two modes in +alternating order, compare full receipts and report wall time and peak memory. +Interpreter and competing load affect timing; independent host measurements +remain outside this package's evidence. + +## Credit + +Credit Tairitsu-Aya for the audit, cases and expected receipt; Josie Elliston +for PR1959's handoff; Kumagi for PR3009 and the remaining parsing lead; and +jackmgaller for the predecessor's `marshal` copying technique. Original invoice, +controller, moment/outer and selected-construction contributors retain their +attribution in the predecessor. No predecessor package is copied or edited. + +Prepared by thakrarsagar with substantial OpenAI Codex assistance, including +separate source inspection, regression-test implementation and code review. +Apache-2.0. No independent human or proof-assistant review is claimed. diff --git a/research/t16-invoice-read-cache/RESULT.json b/research/t16-invoice-read-cache/RESULT.json new file mode 100644 index 0000000000..79bfa2e001 --- /dev/null +++ b/research/t16-invoice-read-cache/RESULT.json @@ -0,0 +1,158 @@ +{ + "python": "3.12.15 (main, Oct 14 2026, 01:05:27) [Clang 23.1.3 ]", + "executable": "/tmp/imb-loop-9963-i01/python/cpython-3.12.15-linux-x86_64-gnu/bin/python3.12", + "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.43", + "verifier_sha256": "6daf8299c4c9e9997b1c200d141a6a2ea9de3946a9a065f956dc977ba797f8cd", + "runs": [ + { + "label": "1-baseline", + "argv": [ + "/tmp/imb-loop-9963-i01/python/cpython-3.12.15-linux-x86_64-gnu/bin/python3.12", + "-B", + "/root/projects/imb-loop-9963/research/t16-invoice-read-cache/verify.py", + "--baseline", + "--receipt", + "/root/projects/imb-loop-9963/.scratch/iteration-01/1-baseline.json" + ], + "wall_seconds": 179.27564533799887, + "user_seconds": 170.425319, + "system_seconds": 8.55583, + "peak_rss_kib": 939892, + "receipt_sha256": "e3c0e3c93fbbc6a63e43502dbb98b9e6556abe6c412e0c142f7a202f0e420541", + "cache": { + "baseline": true, + "hits": 0, + "misses": 0, + "paths": 0, + "retained_bytes": 0 + } + }, + { + "label": "2-cached", + "argv": [ + "/tmp/imb-loop-9963-i01/python/cpython-3.12.15-linux-x86_64-gnu/bin/python3.12", + "-B", + "/root/projects/imb-loop-9963/research/t16-invoice-read-cache/verify.py", + "--receipt", + "/root/projects/imb-loop-9963/.scratch/iteration-01/2-cached.json" + ], + "wall_seconds": 170.21300993097248, + "user_seconds": 160.368821, + "system_seconds": 9.004701, + "peak_rss_kib": 970392, + "receipt_sha256": "e3c0e3c93fbbc6a63e43502dbb98b9e6556abe6c412e0c142f7a202f0e420541", + "cache": { + "baseline": false, + "hits": 488, + "misses": 140, + "paths": 12, + "retained_bytes": 30013304 + } + }, + { + "label": "3-cached", + "argv": [ + "/tmp/imb-loop-9963-i01/python/cpython-3.12.15-linux-x86_64-gnu/bin/python3.12", + "-B", + "/root/projects/imb-loop-9963/research/t16-invoice-read-cache/verify.py", + "--receipt", + "/root/projects/imb-loop-9963/.scratch/iteration-01/3-cached.json" + ], + "wall_seconds": 179.69449680298567, + "user_seconds": 170.101426, + "system_seconds": 9.23581, + "peak_rss_kib": 970320, + "receipt_sha256": "e3c0e3c93fbbc6a63e43502dbb98b9e6556abe6c412e0c142f7a202f0e420541", + "cache": { + "baseline": false, + "hits": 488, + "misses": 140, + "paths": 12, + "retained_bytes": 30013304 + } + }, + { + "label": "4-baseline", + "argv": [ + "/tmp/imb-loop-9963-i01/python/cpython-3.12.15-linux-x86_64-gnu/bin/python3.12", + "-B", + "/root/projects/imb-loop-9963/research/t16-invoice-read-cache/verify.py", + "--baseline", + "--receipt", + "/root/projects/imb-loop-9963/.scratch/iteration-01/4-baseline.json" + ], + "wall_seconds": 197.75562528698356, + "user_seconds": 187.5488, + "system_seconds": 9.366797, + "peak_rss_kib": 939804, + "receipt_sha256": "e3c0e3c93fbbc6a63e43502dbb98b9e6556abe6c412e0c142f7a202f0e420541", + "cache": { + "baseline": true, + "hits": 0, + "misses": 0, + "paths": 0, + "retained_bytes": 0 + } + } + ], + "schema_version": 1, + "observed_on": "2026-10-17", + "base_commit": "463113d675af7410ac91d53a4fed51ec6494eb9a", + "cpu": "AMD EPYC-Genoa Processor; 4 logical CPUs available", + "evidence_scope": "Local observations on one shared Linux host; no confidence interval or portable speed guarantee.", + "measurement": { + "order": [ + "baseline", + "cached", + "cached", + "baseline" + ], + "method": "Sequential fresh subprocesses; time.perf_counter around spawn/wait for wall time and os.wait4 rusage for child user/system CPU and ru_maxrss (KiB on Linux); 600-second alarm per process. No concurrent replay in this worktree.", + "receipt_check": "Every emitted receipt was compared byte for byte with unchanged research/t16-invoice-boundary-audit/EXPECTED.json.", + "scratch": "Absolute executable and receipt paths are session observations, not required locations; use python3.12 and disposable receipt paths when reproducing. Session scratch is deleted after publication." + }, + "aggregate": { + "baseline_mean_wall_seconds": 188.5156353124912, + "cached_mean_wall_seconds": 174.95375336697907, + "mean_wall_reduction_percent": 7.194035615683236, + "baseline_mean_peak_rss_kib": 939848, + "cached_mean_peak_rss_kib": 970356 + }, + "retained_coverage": { + "cases": 54, + "invoice_reject": 28, + "invoice_accept": 26, + "assembly_reject": 16, + "assembly_accept": 10, + "all_four_receipts_byte_identical": true + }, + "focused_controls": { + "command": [ + "python3.12", + "-B", + "research/t16-invoice-read-cache/test_verify.py" + ], + "tests": 10, + "status": "PASS" + }, + "obligations_not_run": [ + "Complete selected cold composition replay", + "All-size supplier or integer multiplication theorem", + "Independent host timing replication" + ], + "registered_replay": { + "command": [ + "python3.12", + "-B", + "scripts/research_package.py", + "--package", + "research/t16-invoice-read-cache" + ], + "completed_at": "2026-10-17T17:56:53Z", + "status": "PASS_REGISTERED_RESEARCH_PACKAGE research/t16-invoice-read-cache outcome=completed", + "focused_tests": 10, + "cases": 54, + "cache_hits": 488, + "cache_misses": 140 + } +} diff --git a/research/t16-invoice-read-cache/SOURCES.json b/research/t16-invoice-read-cache/SOURCES.json new file mode 100644 index 0000000000..60a352b9b1 --- /dev/null +++ b/research/t16-invoice-read-cache/SOURCES.json @@ -0,0 +1,102 @@ +{ + "schema_version": 1, + "base_commit": "463113d675af7410ac91d53a4fed51ec6494eb9a", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "b1fea0c94436b236cc2b7d41a7be032811b94c81", + "sha256": "bb2eaa58647b353f88d0aec2eafb9d30df1be53589231b23e12ceef3e4bad129" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "67f085acd86acd7d4e03922d9cf4b26bca66fed5", + "sha256": "47b42433f5a85809a3aadd29c2c339da2c6ed728e4f7762df6b88bccb7874cf0" + }, + { + "path": "research/t16-invoice-boundary-audit/EXPECTED.json", + "usage": "input", + "git_blob": "6cc1f4795640c09431738332b0a9e9c48b48f796", + "sha256": "e3c0e3c93fbbc6a63e43502dbb98b9e6556abe6c412e0c142f7a202f0e420541" + }, + { + "path": "research/t16-invoice-boundary-audit/SOURCES.json", + "usage": "input", + "git_blob": "21c8fa5b9ee76819e40de3f1473077cd02f4dc7f", + "sha256": "ef816261008c212dbde2498722f66e37555944f206be061fa8c92dc72a22fb9d" + }, + { + "path": "research/t16-invoice-boundary-audit/VERIFY.json", + "usage": "input", + "git_blob": "425dbc047d76503857973db43c0f6dce53a2d061", + "sha256": "aedf0434c96fb814b853e12ad4cfc7a05bd1a8b324da8666cf1b9e4d75228026" + }, + { + "path": "research/t16-invoice-boundary-audit/cases.py", + "usage": "input", + "git_blob": "b0dfceedc6b14d5856b9d232d11aa022cd29b71e", + "sha256": "8f7a8a583b691adbc5f77fe5826c41e299d0d7bbab79bbf79122039a712c46fb" + }, + { + "path": "research/t16-invoice-boundary-audit/verify.py", + "usage": "input", + "git_blob": "5669c919f7f26b2db3ef6140359fd13c99210c74", + "sha256": "fdd9c9f2800f4ea53f241e05bf816587faa4e812879b679d597b7734058d664b" + }, + { + "path": "research/community-frontier1094-audit/selected-replay.tar.gz", + "usage": "input", + "git_blob": "c24daf067f9c543421d89ca2eeaa4eb78a57fb6f", + "sha256": "f1da80610bf33c58527f18ad63b4350d0b864e1d4486141ee17ac9bed4d529d6" + }, + { + "path": "research/complex-k492-owner-repair/vendor/multicold/engines/base_two_moment.py", + "usage": "input", + "git_blob": "14a90fa0e788217856f070382c9e61bd7c603c61", + "sha256": "aee70c242a2f1d225528fabe2002e1564d1dc884797c07d28a831c7878ee9098" + }, + { + "path": "research/complex-k492-owner-repair/vendor/multicold/engines/moment.py", + "usage": "input", + "git_blob": "7d697cb45203fe72d97037403d6da41f71f3b7fd", + "sha256": "81cbc864e106418fc7854322746f72180cb8110ff39f74450a3b8d55a82c1482" + }, + { + "path": "research/complex-k492-owner-repair/vendor/multicold/engines/outer.py", + "usage": "input", + "git_blob": "43007488ad2c43187c77cc6798beff8b1cd76ad2", + "sha256": "c1ebb52163a7219504c728a5c2b098bf1e50bf20148a2879c5a74c3cadc7c03c" + }, + { + "path": "research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/evidence/audit/PREFIX-AND-INVOICE.json", + "usage": "input", + "git_blob": "a16bf40bccc81278a443e0d5f09b44495035de81", + "sha256": "ce1bfa49be88ae67a404f606a11db2232b7e1f2610d4418058e39749d86d8478" + }, + { + "path": "research/complex-t16-rare-pattern-exchanges/data/OWNERS.json.gz", + "usage": "input", + "git_blob": "d3ba33d7e703a3229d9fb8a360f69d2f318bbb70", + "sha256": "87902c528dfab7d503dcf8a784260e3ab6fb7424c418a1820f7261f25be7b6bc" + }, + { + "path": "research/complex-t16-rare-pattern-exchanges/data/PLAN.json", + "usage": "input", + "git_blob": "f1bec76e8e9b7a1f09c1e05cb6cce973b6e59f1b", + "sha256": "7bd786ba8e66408f02aa9ef446eaf20c1bfe981ea2e4dd1e5a929f0aa7767bc0" + }, + { + "path": "research/complex-t16-rare-pattern-exchanges/data/SPEC.json", + "usage": "input", + "git_blob": "af013ccc2975d9d575d116f8a463cc947f220c8d", + "sha256": "8cb51f25a5da4a29d3783853025b73ff1f452429d4fdb30c599e598bcab0e71d" + }, + { + "path": "research/complex-t16-rare-pattern-exchanges/upstream1087.tar.gz", + "usage": "input", + "git_blob": "2970ef79b0441e937449bed30f68f2e0558186ba", + "sha256": "d4acf04daef85d4514f80ec5fcd2492d0c60e5d6c49fa71b3e6523f8db2dea86" + } + ] +} diff --git a/research/t16-invoice-read-cache/TASK.json b/research/t16-invoice-read-cache/TASK.json new file mode 100644 index 0000000000..45fe6a94a6 --- /dev/null +++ b/research/t16-invoice-read-cache/TASK.json @@ -0,0 +1,95 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 6, + "base_commit": "463113d675af7410ac91d53a4fed51ec6494eb9a", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Read the selected reproduction command and choose a single replay stage for profiling.", + "scope": "Positive control / source-bound orientation only; this is not a new construction or full cold replay." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "refined", + "actual_scope": "Speed up the T16 invoice boundary audit by reusing parsed JSON for byte-identical input files while returning fresh mutable values; retain the full 54-case receipt.", + "scope_change_reason": "Takes the remaining repeated-JSON-parsing lead of Kumagi PR3009 after checking that the compression optimization is already on main.", + "bottleneck_link": "Faster complete boundary replays improve review throughput without dropping source binding or any admission predicate.", + "budget_minutes": 60, + "stop_rule": "Stop after same-fixture timing and complete receipt comparison, with changed-input and mutable-return controls; abandon acceleration if it is not beneficial.", + "outcome": "completed", + "evidence": "RESULT.json: four fresh Python 3.12.15 processes in baseline/cached/cached/baseline order; all complete 54-case receipts byte-identical to the original. Mean wall time 188.516 -> 174.954 seconds (7.19% lower locally), mean peak RSS 939848 -> 970356 KiB. Cached runs reuse 488 of 628 reads; ten invalidation/value-isolation controls pass. Shared-host timing ranges overlap; no portable speedup claim. Registered package replay also passed on Python 3.12.15.", + "author": "thakrarsagar with OpenAI Codex", + "ai_assistance": "Substantial OpenAI Codex assistance in implementation, profiling, tests, source inspection and writing; parallel read-only source review and separate regression-test implementation. No independent human review claimed.", + "open_questions": [ + { + "question": "Does this reader reuse improve replay on another supported Python 3.12 host?", + "why_it_matters": "The observed speedup depends on interpreter, CPU and competing load; mathematical coverage does not.", + "first_step": "Run the baseline and cached commands in alternating order, compare full receipts and report elapsed time and peak memory.", + "blocked_by": "No correctness blocker; independent host measurements are not included here." + } + ], + "coordination": { + "claim": "https://github.com/CrocSwap/integer-mult-bounds/discussions/2051#discussioncomment-19001801", + "entry_upstream": "463113d675af7410ac91d53a4fed51ec6494eb9a", + "entry_utc": "2026-10-17T17:22:14Z", + "priority_revision": 6, + "overlap": { + "open_prs": 249, + "discussion_records": 1928, + "complete_pagination": true, + "warnings": [] + }, + "predecessor_pr": 3009, + "predecessor_head": "95568ae83a73ace578e3972a01aef55c889e58a2", + "submission_refresh": { + "upstream": "463113d675af7410ac91d53a4fed51ec6494eb9a", + "checked_at": "2026-10-17T17:51:05.234805+00:00", + "priority_revision": 6, + "coverage": { + "open_prs": { + "fetched_at": "2026-10-17T17:50:37.111280+00:00", + "complete_pagination": true, + "records": 253 + }, + "discussions": { + "fetched_at": "2026-10-17T17:51:04.981869+00:00", + "complete_pagination": true, + "records": 2017 + } + }, + "warnings": [], + "finding": "No competing T16 reader-cache PR or claim found; predecessor compression optimization already incorporated. No matches is not a novelty certificate.", + "upstream_checked_at": "2026-10-17T17:56:41Z" + } + } +} diff --git a/research/t16-invoice-read-cache/VERIFY.json b/research/t16-invoice-read-cache/VERIFY.json new file mode 100644 index 0000000000..12fe78a099 --- /dev/null +++ b/research/t16-invoice-read-cache/VERIFY.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "research/community-frontier1094-audit/selected-replay.tar.gz", + "research/complex-k492-owner-repair/vendor/multicold/engines/base_two_moment.py", + "research/complex-k492-owner-repair/vendor/multicold/engines/moment.py", + "research/complex-k492-owner-repair/vendor/multicold/engines/outer.py", + "research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/evidence/audit/PREFIX-AND-INVOICE.json", + "research/complex-t16-rare-pattern-exchanges/data/OWNERS.json.gz", + "research/complex-t16-rare-pattern-exchanges/data/PLAN.json", + "research/complex-t16-rare-pattern-exchanges/data/SPEC.json", + "research/complex-t16-rare-pattern-exchanges/upstream1087.tar.gz", + "research/t16-invoice-boundary-audit/EXPECTED.json", + "research/t16-invoice-boundary-audit/SOURCES.json", + "research/t16-invoice-boundary-audit/VERIFY.json", + "research/t16-invoice-boundary-audit/cases.py", + "research/t16-invoice-boundary-audit/verify.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-B", + "{package}/test_verify.py" + ], + "timeout_seconds": 120, + "checks": "Ten controls for fresh nested values, byte invalidation despite preserved metadata, repeated uncached mutations and restoration, malformed JSON/gzip, missing files, changed source pins and rejection of optimized replay." + }, + { + "argv": [ + "{python}", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 600, + "checks": "Authenticate all predecessor code and transitive inputs; execute every predicate in the predecessor boundary audit and its 54 real-file mutation cases with the cached JSON reader; compare the complete reconstructed receipt to unchanged EXPECTED.json." + } + ] +} diff --git a/research/t16-invoice-read-cache/test_verify.py b/research/t16-invoice-read-cache/test_verify.py new file mode 100644 index 0000000000..2b3ed061f0 --- /dev/null +++ b/research/t16-invoice-read-cache/test_verify.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 thakrarsagar. Developed with OpenAI Codex. +"""Controls for fresh reads, independent return values, and replay boundaries.""" + +import gzip +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from verify import JsonReader, authenticate + + +class JsonReaderTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) + + def test_plain_and_gzip_json_types_and_warm_hits(self): + values = [None, False, 10**80, 1.25, "λ", [], {"rows": [True, None]}] + reader = JsonReader() + for suffix in (".json", ".json.gz"): + for index, expected in enumerate(values): + with self.subTest(suffix=suffix, value=expected): + path = self.directory / f"value-{index}{suffix}" + raw = json.dumps(expected).encode() + path.write_bytes(gzip.compress(raw) if suffix.endswith(".gz") else raw) + for _ in range(2): + actual = reader(path) + self.assertEqual(actual, expected) + self.assertIs(type(actual), type(expected)) + self.assertEqual(reader.misses, 2 * len(values)) + self.assertEqual(reader.hits, 2 * len(values)) + + def test_same_size_same_mtime_replacement_and_restoration(self): + for suffix in (".json", ".json.gz"): + with self.subTest(suffix=suffix): + reader = JsonReader() + path = self.directory / f"changing{suffix}" + original, changed = b'{"rows":[1]}', b'{"rows":[2]}' + if suffix.endswith(".gz"): + original = gzip.compress(original, mtime=0) + changed = gzip.compress(changed, mtime=0) + self.assertEqual(len(original), len(changed)) + path.write_bytes(original) + before = path.stat() + self.assertEqual(reader(path), {"rows": [1]}) + for raw, expected in ((changed, 2), (original, 1)): + path.write_bytes(raw) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + self.assertEqual(path.stat().st_mtime_ns, before.st_mtime_ns) + self.assertEqual(reader(path), {"rows": [expected]}) + self.assertEqual(reader.misses, 2) + self.assertEqual(reader.hits, 1) + self.assertEqual(len(reader.cache), 1) + self.assertEqual(reader(path), {"rows": [1]}) + self.assertEqual(reader.hits, 2) + + def test_changed_content_is_fresh_and_original_remains_cached(self): + for suffix in (".json", ".json.gz"): + with self.subTest(suffix=suffix): + reader = JsonReader() + path = self.directory / f"alternating{suffix}" + original = b'{"rows":[{"values":[1]}]}' + changed = b'{"rows":[{"values":[2]}]}' + if suffix.endswith(".gz"): + original = gzip.compress(original, mtime=0) + changed = gzip.compress(changed, mtime=0) + path.write_bytes(original) + self.assertEqual(reader(path), {"rows": [{"values": [1]}]}) + path.write_bytes(changed) + first_changed = reader(path) + self.assertEqual(first_changed, {"rows": [{"values": [2]}]}) + first_changed["rows"][0]["values"].append(99) + self.assertEqual(reader(path), {"rows": [{"values": [2]}]}) + path.write_bytes(original) + self.assertEqual(reader(path), {"rows": [{"values": [1]}]}) + self.assertEqual(reader.misses, 3) + self.assertEqual(reader.hits, 1) + self.assertEqual(len(reader.cache), 1) + + def test_cold_and_warm_returns_have_independent_nested_values(self): + for suffix in (".json", ".json.gz"): + with self.subTest(suffix=suffix): + reader = JsonReader() + path = self.directory / f"mutable{suffix}" + expected = {"rows": [{"values": [1]}]} + raw = json.dumps(expected).encode() + path.write_bytes(gzip.compress(raw) if suffix.endswith(".gz") else raw) + for _ in range(3): + actual = reader(path) + self.assertEqual(actual, expected) + actual["rows"][0]["values"].append(99) + actual["rows"].append({"extra": True}) + actual["added"] = "caller-owned" + self.assertEqual(reader.misses, 1) + self.assertEqual(reader.hits, 2) + + def test_malformed_content_after_warming_is_rejected(self): + plain = b'{"value":1}' + compressed = gzip.compress(plain, mtime=0) + cases = [ + ("bad.json", plain, b"{", json.JSONDecodeError), + ("bad-json.json.gz", compressed, gzip.compress(b"{"), json.JSONDecodeError), + ("bad-gzip.json.gz", compressed, b"not gzip", gzip.BadGzipFile), + ("truncated.json.gz", compressed, compressed[:-4], EOFError), + ] + for name, original, invalid, error in cases: + with self.subTest(name=name): + reader = JsonReader() + path = self.directory / name + path.write_bytes(original) + self.assertEqual(reader(path), {"value": 1}) + self.assertEqual(reader(path), {"value": 1}) + path.write_bytes(invalid) + for _ in range(2): + with self.assertRaises(error): + reader(path) + path.write_bytes(original) + self.assertEqual(reader(path), {"value": 1}) + + def test_missing_warmed_path_is_rejected(self): + reader = JsonReader() + path = self.directory / "removed.json" + path.write_bytes(b"[]") + self.assertEqual(reader(path), []) + self.assertEqual(reader(path), []) + path.unlink() + with self.assertRaises(FileNotFoundError): + reader(path) + + def test_paths_have_separate_entries(self): + reader = JsonReader() + first, second = self.directory / "first.json", self.directory / "second.json" + first.write_bytes(b"[1]") + second.write_bytes(b"[1]") + self.assertEqual(reader(first), [1]) + self.assertEqual(reader(second), [1]) + second.write_bytes(b"[2]") + self.assertEqual(reader(second), [2]) + self.assertEqual(reader(first), [1]) + self.assertEqual(len(reader.cache), 2) + self.assertEqual(reader.misses, 3) + self.assertEqual(reader.hits, 1) + + def test_suffix_controls_gzip_decoding(self): + reader = JsonReader() + plain = self.directory / "plain.json" + compressed = self.directory / "compressed.json.gz" + plain.write_bytes(b"[1]") + compressed.write_bytes(b"[1]") + self.assertEqual(reader(plain), [1]) + with self.assertRaises(gzip.BadGzipFile): + reader(compressed) + plain.write_bytes(gzip.compress(b"[1]")) + with self.assertRaises(UnicodeDecodeError): + reader(plain) + + def test_authenticate_rejects_changed_pinned_input(self): + sources = json.loads(Path(__file__).with_name("SOURCES.json").read_bytes()) + pin = next(pin for pin in sources["files"] if pin["usage"] == "input") + changed = self.directory / pin["path"] + changed.parent.mkdir(parents=True) + changed.write_bytes(b"changed input\n") + with self.assertRaises(ValueError) as error: + authenticate(root=self.directory) + self.assertEqual(str(error.exception), "Input changed: " + pin["path"]) + + def test_optimized_python_is_rejected(self): + result = subprocess.run( + [sys.executable, "-O", "-B", str(Path(__file__).resolve().with_name("verify.py"))], + cwd=self.directory, capture_output=True, text=True, timeout=5, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Assertions must remain enabled", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/research/t16-invoice-read-cache/verify.py b/research/t16-invoice-read-cache/verify.py new file mode 100644 index 0000000000..8c16b4183f --- /dev/null +++ b/research/t16-invoice-read-cache/verify.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Source-bound T16 replay with fresh-value JSON reuse. + +thakrarsagar with substantial OpenAI Codex assistance. Apache-2.0. +Tairitsu-Aya's audit and Kumagi's compression improvement remain unchanged. +""" + +import argparse +import gzip +import hashlib +import json +import marshal +from pathlib import Path +import sys +import tempfile +import types + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +PREDECESSOR = ROOT / "research/t16-invoice-boundary-audit" + + +class JsonReader: + """Reuse first-seen contents when fixtures return; always give fresh values.""" + + def __init__(self): + self.cache = {} + self.hits = self.misses = 0 + + def __call__(self, path): + raw = path.read_bytes() + previous = self.cache.get(path) + if previous is not None and previous[0] == raw: + self.hits += 1 + return marshal.loads(previous[1]) + value = json.loads(gzip.decompress(raw) if path.suffix == ".gz" else raw) + if previous is None: + self.cache[path] = (raw, marshal.dumps(value)) + self.misses += 1 + return value + + +def authenticate(root=ROOT): + for pin in json.loads((HERE / "SOURCES.json").read_bytes())["files"]: + if pin["usage"] == "input": + actual = hashlib.sha256((root / pin["path"]).read_bytes()).hexdigest() + if actual != pin["sha256"]: + raise ValueError("Input changed: " + pin["path"]) + + +def load_source(name, path): + module = types.ModuleType(name) + module.__file__ = str(path) + exec(compile(path.read_bytes(), str(path), "exec", optimize=0), module.__dict__) + return module + + +def replay(baseline=False): + authenticate() + sys.modules["cases"] = load_source("cases", PREDECESSOR / "cases.py") + audit = load_source("t16_boundary", PREDECESSOR / "verify.py") + reader = JsonReader() + original_load = audit.load + + def load(name, path): + module = original_load(name, path) + if name == "frozen_invoice" and not baseline: + module.read = reader + return module + + audit.load = load + with tempfile.TemporaryDirectory(prefix="t16-read-cache-") as directory: + actual = audit.run(Path(directory)) + expected = json.loads((PREDECESSOR / "EXPECTED.json").read_bytes()) + if actual != expected: + raise ValueError("Full T16 receipt differs from predecessor") + return actual, reader + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", action="store_true", help="Use the original reader") + parser.add_argument("--receipt", type=Path, help="Write the reconstructed full receipt") + args = parser.parse_args() + if not __debug__: + raise RuntimeError("Assertions must remain enabled for the frozen checkers") + # Direct invocations also ignore preexisting bytecode for frozen module imports. + with tempfile.TemporaryDirectory(prefix="t16-read-pycache-") as directory: + sys.pycache_prefix = directory + actual, reader = replay(args.baseline) + if args.receipt is not None: + args.receipt.write_text(json.dumps(actual, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", newline="\n") + print(actual["status"], json.dumps(actual["summary"], sort_keys=True)) + print("PASS_T16_READ_CACHE", json.dumps({"baseline": args.baseline, + "hits": reader.hits, "misses": reader.misses, "paths": len(reader.cache), + "retained_bytes": sum(len(raw) + len(value) for raw, value in reader.cache.values())}, + sort_keys=True)) + + +if __name__ == "__main__": + main()