From 12cf382574627610bda83da3d4c5993fbd4bab50 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Wed, 28 Oct 2026 16:20:51 +0000 Subject: [PATCH 1/2] perf(research): bound working-source comparison memory --- .../source-comparison-memory/BENCHMARK.json | 117 ++++++++++++++ research/source-comparison-memory/PROOF.md | 53 +++++++ research/source-comparison-memory/README.md | 115 ++++++++++++++ .../source-comparison-memory/SOURCES.json | 30 ++++ research/source-comparison-memory/TASK.json | 65 ++++++++ research/source-comparison-memory/VERIFY.json | 38 +++++ .../source-comparison-memory/benchmark.py | 98 ++++++++++++ research/source-comparison-memory/verify.py | 34 +++++ scripts/research_package.py | 24 ++- tests/test_research_source_comparison.py | 144 ++++++++++++++++++ 10 files changed, 717 insertions(+), 1 deletion(-) create mode 100644 research/source-comparison-memory/BENCHMARK.json create mode 100644 research/source-comparison-memory/PROOF.md create mode 100644 research/source-comparison-memory/README.md create mode 100644 research/source-comparison-memory/SOURCES.json create mode 100644 research/source-comparison-memory/TASK.json create mode 100644 research/source-comparison-memory/VERIFY.json create mode 100644 research/source-comparison-memory/benchmark.py create mode 100644 research/source-comparison-memory/verify.py create mode 100644 tests/test_research_source_comparison.py diff --git a/research/source-comparison-memory/BENCHMARK.json b/research/source-comparison-memory/BENCHMARK.json new file mode 100644 index 0000000000..44bf3518c7 --- /dev/null +++ b/research/source-comparison-memory/BENCHMARK.json @@ -0,0 +1,117 @@ +{ + "schema_version": 1, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "package": "research/physical-gaussian-core", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "load_average": [ + 5.76220703125, + 3.64111328125, + 2.79833984375 + ], + "logical_cpus": 18, + "baseline_sha256": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6", + "candidate_sha256": "791593391551c8d4e40168dacb7ae3cf2d0eb00108944bf62cf1cd95278a8b10", + "benchmark_sha256": "ec9dab5504632fa7938925f6a34299012a47a88eb073c9bc323a5e3660aa97ff", + "failures": 0, + "pairs": 5, + "samples": [ + { + "pair": 0, + "side": "baseline", + "seconds": 0.25392808415926993, + "peak_traced_bytes": 127510623, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 0, + "side": "candidate", + "seconds": 0.33391400007531047, + "peak_traced_bytes": 74011708, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 1, + "side": "baseline", + "seconds": 0.2253677500411868, + "peak_traced_bytes": 127510623, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 1, + "side": "candidate", + "seconds": 0.33021320798434317, + "peak_traced_bytes": 74011826, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 2, + "side": "baseline", + "seconds": 0.2261271660681814, + "peak_traced_bytes": 127510623, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 2, + "side": "candidate", + "seconds": 0.3400371668394655, + "peak_traced_bytes": 74011826, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 3, + "side": "baseline", + "seconds": 0.238344790879637, + "peak_traced_bytes": 127510623, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 3, + "side": "candidate", + "seconds": 0.3299693330191076, + "peak_traced_bytes": 74011826, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 4, + "side": "baseline", + "seconds": 0.232870833016932, + "peak_traced_bytes": 127510623, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + }, + { + "pair": 4, + "side": "candidate", + "seconds": 0.3290826668962836, + "peak_traced_bytes": 74011826, + "result_sha256": "01a4d1aa3f93e76e0a4f9dc8304885009774a6fcc0c18e4abd77dcbb73346078" + } + ], + "summary": { + "baseline": { + "seconds": { + "median": 0.232870833016932, + "min": 0.2253677500411868, + "max": 0.25392808415926993 + }, + "peak_traced_bytes": { + "median": 127510623, + "min": 127510623, + "max": 127510623 + } + }, + "candidate": { + "seconds": { + "median": 0.33021320798434317, + "min": 0.3290826668962836, + "max": 0.3400371668394655 + }, + "peak_traced_bytes": { + "median": 74011826, + "min": 74011708, + "max": 74011826 + } + } + }, + "scope": "Fresh-process complete metadata/pin validation; warm OS cache; traced Python allocation, not RSS. No scientific verifier." +} diff --git a/research/source-comparison-memory/PROOF.md b/research/source-comparison-memory/PROOF.md new file mode 100644 index 0000000000..87cbd21f4d --- /dev/null +++ b/research/source-comparison-memory/PROOF.md @@ -0,0 +1,53 @@ +# Exact comparison with bounded auxiliary memory + +The input contract is the existing validator contract. `relative_file` has +already checked the repository-relative path and produced a regular file. +The working checkout remains stable throughout validation. Write N for its +byte length and B for the retained pinned Git blob. This proof concerns the +new comparison, not the correctness of the blob reader or any mathematics in +an input package. + +`file_equals_bytes` opens the file in binary mode, obtains N from the opened +file descriptor, and allocates a buffer of capacity + + q = min(1048576, max(1, N)). + +It keeps memory views of that buffer and B. The loop has exactly ceil(N/q) +iterations, zero when N is zero. Its iteration at offset j*q reads into the +same buffer. For a stable regular file, the read count is +`min(q, N-j*q)`, and the buffer's first count bytes are precisely that part +of the file. The corresponding slice of B is compared without allocating +a second bytes object. A size mismatch, count mismatch, or unequal slice +sets `matches` false permanently. No mismatch exits the loop early. + +If the comparison returns true, the lengths agree, every successive slice +agrees and the final one-byte read found EOF. Thus every byte agrees with B. +Conversely, if the file equals B, all lengths and slices agree, and the final +read finds EOF, so the comparison returns true. This includes an empty file, +embedded zero bytes and a final partial buffer. Therefore, under the stable +regular-file contract, the predicate is exactly the former +`B == path.read_bytes()` predicate. + +The existing blob-ID and SHA256 checks still run first. The caller still +raises `ValueError("working source drift: " + name)` on a false result. +Open, fstat, readinto, final-read and close errors propagate through the +existing validator. The file context closes on every exit. Because the +whole initial extent is read even after a mismatch, an I/O error in a later +initial-extent chunk is not suppressed by an earlier unequal chunk. The +existing outer `closing(git_blobs(...))` still closes the source reader when +any comparison fails. + +The comparison's additional data buffer is at most 1 MiB, independently of +N and len(B). Memory views and loop integers add constant-size interpreter +objects at the supported implementation sizes. The pinned blob itself is +still retained, and the Git reader can retain other objects. No bound on +total validator RSS or Python allocator internals follows from this claim. +The old comparison allocated N additional bytes for a matching working +file. The measured reduction in the full validator is smaller than that +local saving because the rest of validation still allocates memory. + +Concurrent mutation is outside the existing stable-checkout contract. The +final read detects some growth and count checks detect some shrinkage; they +do not make validation an atomic filesystem snapshot. No stronger concurrent +mutation guarantee, universal speedup or fixed-allocation Python runtime is +claimed. diff --git a/research/source-comparison-memory/README.md b/research/source-comparison-memory/README.md new file mode 100644 index 0000000000..2708ddab02 --- /dev/null +++ b/research/source-comparison-memory/README.md @@ -0,0 +1,115 @@ +# Lower-memory exact research-source validation + +This contribution was produced by Nick Petalas. + +The shared package validator now compares each working source with its pinned +Git bytes using a reusable buffer of at most 1 MiB. It keeps exact byte equality, +blob/SHA256 checks, source-drift diagnostics and file/reader cleanup. The change +adds one helper and replaces one comparison in `scripts/research_package.py`. +[PROOF.md](PROOF.md) proves equivalence under the existing stable regular-file +checkout contract. This is a verifier-engineering result with a scoped proof +and finite tests. It is not a new multiplication theorem or kappa improvement. + +On the complete `research/physical-gaussian-core` metadata/pin validation, +peak traced Python allocation falls from 127,510,623 bytes to a median +74,011,826 bytes, about 42%. Five alternating fresh-process pairs all return +the same complete task/manifest result. There are zero failed validations. +The fixture includes the unchanged 54,548,480-byte `word-export.tar` live +input; its scientific verifier is not run or audited. + +The memory saving has a time cost. Baseline validation takes a median +0.232871 seconds, range 0.225368 to 0.253928. The changed validation takes +0.330213 seconds, range 0.329083 to 0.340037. These are warm-cache local +measurements on macOS arm64 with CPython 3.12.15. They are traced runs, +not ordinary uninstrumented runtime measurements, cold-cache timings, +RSS measurements or a claim about whole scientific replay speed. +[BENCHMARK.json](BENCHMARK.json) retains every sample, environment, exact +source hash and complete-output hash. The file was generated by the benchmark. + +## Reproduce + +From the repository root, use CPython 3.12 and the standard library: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/source-comparison-memory/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -O -W error -B research/source-comparison-memory/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/source-comparison-memory +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/source-comparison-memory --check-source-history +``` + +Set `TMPDIR` to an external scratch directory for the temporary fixture +repositories. Elsewhere substitute the local Python 3.12 path. The marker is +`PASS_SOURCE_COMPARISON_MEMORY`. Each direct run generates a receipt on stdout +with the current validator/test/verifier hashes; it writes no receipt file. +Seven tests cover binary/empty/chunk-boundary files, same-size changes, +truncation and append, open errors, late I/O failure after an earlier mismatch, +closure, restored live inputs and unchanged blob/SHA rejection. Assertions in +the test framework remain active under optimized Python. + +For the optional historical benchmark, first materialize its input paths in +a sparse checkout, then use a new external output directory: + +```sh +git sparse-checkout add research/physical-gaussian-core references/semantic-bulk/rad20 +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/source-comparison-memory/benchmark.py --work /absolute/new/scratch --pairs 5 +``` + +The benchmark reads the baseline validator from canonical main +`721c03822c421b7f7fcbb2598993b947c4d88419`, imports the working candidate, +and times the complete `validate` call inside each fresh child process. +Every result must have the same full-output SHA256. The benchmark is opt-in; +CI replays the behavioral contracts and does not enforce host timing numbers. +The normal research-tooling test command also discovers the new tests: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -W error -B -m unittest discover -s tests -p 'test_research_*.py' -v +``` + +## Scope and implementation limits + +The retained pinned Git blob is still allocated in full. Traced peak allocation +for complete validation includes that blob and other validator state. Only +the working-file comparison's added data buffer is bounded by 1 MiB. Managed +Python allocation remains a Power-of-Ten rule-3 deviation; no fixed-allocation +runtime is claimed. Each comparison has `ceil(N/q)` iterations for the opened +file's initial size N and buffer capacity q. It reads the initial extent even +after a mismatch. Context managers retain file and Git-reader cleanup. +Functions added here remain under 60 physical lines. + +The source loader used by the benchmark and verifier imports fixed local Python +files through the standard import machinery. This bounded test infrastructure +is a documented dynamic-dispatch deviation; the production comparison has no +dynamic code, recursion, subprocesses or mutable shared state. The benchmark +uses the existing Git history and at most 20 bounded child processes. It does +not create another project checkout. Its temporary baseline file and outputs +must stay outside the repository. + +Concurrent file mutation remains outside the stable-checkout contract. Native +Windows execution, whole-process RSS, scientific package replay and selected +frontier replay are unrun. Every inherited mathematical and physical obligation +remains separate. Shared-runner changes select every registered research +package in the existing CI planner; no historical scientific suites are run +locally for this change. + +## Open next questions + +Can the pinned Git blob also be streamed while preserving exact provenance, +SHA256, response framing, final process status and cleanup on early failure? +That could remove the remaining whole-blob allocation. First specify a bounded +chunk interface and every existing consumer. The current `git_blobs` contract +yields complete bytes, so changing only the working comparison does not settle +that question. + +## Credit + +Douglas Colkitt supplies the package runner. woklerx/PR1452, +Arham-Tahir64/PR1595 and ywz65/PR2539 supply the retained batched source-reader +and draft-generation lineage. The later Git-response deadline, replacement-ref, +path and descendant-cleanup contributors retain their source attribution. +JoyboyBrian/PR6087 separately reuses planner input-path validation and does not +change source-byte comparison. Jose Manuel/jmbarrancoml supplies the unchanged +physical-gaussian-core benchmark package and its source notices. This work +changes none of those credit lines or source packages. Buffered file I/O and +memory views are standard techniques; no literature-priority claim is made. +New work is Apache-2.0. Independent human and formal proof-kernel review are +unclaimed. diff --git a/research/source-comparison-memory/SOURCES.json b/research/source-comparison-memory/SOURCES.json new file mode 100644 index 0000000000..fd0ed88b4f --- /dev/null +++ b/research/source-comparison-memory/SOURCES.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/research_package.py", + "usage": "provenance", + "git_blob": "48fe9feb4699c09e95505d95c327ea998ac8b3a7", + "sha256": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6" + }, + { + "path": "scripts/new_research_package.py", + "usage": "provenance", + "git_blob": "1bd348ad3c27a8d0addac6156171e589edf4ff08", + "sha256": "a9a66cb591239c27928047146040cb9adb8d138fdc327f2fc8cb873ef9892c27" + } + ] +} diff --git a/research/source-comparison-memory/TASK.json b/research/source-comparison-memory/TASK.json new file mode 100644 index 0000000000..41a8d1cd1a --- /dev/null +++ b/research/source-comparison-memory/TASK.json @@ -0,0 +1,65 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "changed", + "actual_scope": "Reduce complete research-package source-validation peak allocation with an exact bounded-memory working-file comparison.", + "scope_change_reason": "The mathematical card allocation is changed to a distinct verifier-engineering bottleneck found in the shared metadata/pin validator. This changes implementation rather than auditing predecessor mathematics.", + "bottleneck_link": "Large live source archives currently allocate a second full working-file buffer during exact pin validation; reducing it lowers the memory needed to reproduce research packages.", + "budget_minutes": 60, + "stop_rule": "Ship the first verified reduction in complete validator peak traced allocation, with identical full validation output and rejecting controls; retain any measured timing regression.", + "outcome": "completed", + "evidence": "Seven normal/optimized contract tests and five alternating fresh-process pairs on complete physical-gaussian-core metadata/pin validation. Raw measured samples and source hashes are in BENCHMARK.json.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Can the retained pinned Git blob also be streamed without weakening provenance and cleanup checks?", + "why_it_matters": "The comparison now uses at most a 1 MiB data buffer, but the existing Git reader still retains a whole blob.", + "first_step": "Design a bounded chunk protocol with exact size, blob identity, SHA256, final status and reader-cleanup contracts before benchmarking it.", + "blocked_by": "The current git_blobs public contract yields complete bytes; changing it affects several consumers." + } + ], + "claim_url": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19246848", + "upstream_refresh_utc": "2026-10-28T16:02:32Z", + "playbook": "Feature", + "throughput_checkpoint": "One writer, one existing isolated worktree. Implement one helper and caller substitution, prove exact comparison, run contracts, then measure the complete metadata/pin consumer serially. No delegated work or scientific replay." +} diff --git a/research/source-comparison-memory/VERIFY.json b/research/source-comparison-memory/VERIFY.json new file mode 100644 index 0000000000..f2695ca40a --- /dev/null +++ b/research/source-comparison-memory/VERIFY.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/research_package.py", + "scripts/new_research_package.py", + "tests/test_research_source_comparison.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 30, + "checks": "Seven exact comparison and complete validator contracts, including binary chunk boundaries, truncation/append, same-size drift, late I/O errors/close, restored inputs and unchanged blob/SHA rejection." + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 30, + "checks": "The same complete contracts under optimized Python, with executable unittest checks still active." + } + ] +} diff --git a/research/source-comparison-memory/benchmark.py b/research/source-comparison-memory/benchmark.py new file mode 100644 index 0000000000..7b36ce12ae --- /dev/null +++ b/research/source-comparison-memory/benchmark.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Measure complete metadata/pin validation, without running scientific checks. + +Produced by Nick Petalas. Apache-2.0. +""" +import argparse +import gc +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import time +import tracemalloc + +ROOT = Path(__file__).resolve().parents[2] +BASE = "721c03822c421b7f7fcbb2598993b947c4d88419" +PACKAGE = "research/physical-gaussian-core" + + +def worker(source): + spec = importlib.util.spec_from_file_location("measured_validator", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + gc.collect() + tracemalloc.start() + start = time.perf_counter() + result = module.validate(ROOT, PACKAGE) + seconds = time.perf_counter() - start + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + encoded = json.dumps(result, sort_keys=True, separators=(",", ":")).encode() + return {"seconds": seconds, "peak_traced_bytes": peak, + "result_sha256": hashlib.sha256(encoded).hexdigest()} + + +def measure(work, pairs): + work.mkdir(parents=True, exist_ok=True) + baseline = work / "baseline_research_package.py" + raw = subprocess.run(["git", "show", BASE + ":scripts/research_package.py"], + cwd=ROOT, check=True, capture_output=True, timeout=30).stdout + baseline.write_bytes(raw) + candidate = ROOT / "scripts/research_package.py" + samples = [] + command = [sys.executable, "-I", "-W", "error", "-B", str(Path(__file__).resolve())] + for pair in range(pairs): + for side, source in (("baseline", baseline), ("candidate", candidate)): + completed = subprocess.run(command + ["--worker", str(source)], cwd=ROOT, + check=True, capture_output=True, text=True, timeout=120) + row = json.loads(completed.stdout) + samples.append({"pair": pair, "side": side, **row}) + signatures = {row["result_sha256"] for row in samples} + if len(signatures) != 1: + raise ValueError("complete validation outputs differ") + summaries = {} + for side in ("baseline", "candidate"): + rows = [row for row in samples if row["side"] == side] + summaries[side] = {key: {"median": statistics.median(r[key] for r in rows), + "min": min(r[key] for r in rows), + "max": max(r[key] for r in rows)} + for key in ("seconds", "peak_traced_bytes")} + return {"schema_version": 1, "base_commit": BASE, "package": PACKAGE, + "python": sys.version, "platform": platform.platform(), + "load_average": os.getloadavg(), "logical_cpus": os.cpu_count(), + "baseline_sha256": hashlib.sha256(raw).hexdigest(), + "candidate_sha256": hashlib.sha256(candidate.read_bytes()).hexdigest(), + "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "failures": 0, "pairs": pairs, "samples": samples, "summary": summaries, + "scope": "Fresh-process complete metadata/pin validation; warm OS cache; traced Python allocation, not RSS. No scientific verifier."} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work", type=Path) + parser.add_argument("--pairs", type=int, default=5, choices=range(5, 11)) + parser.add_argument("--worker", type=Path, help=argparse.SUPPRESS) + args = parser.parse_args() + if args.worker is not None: + print(json.dumps(worker(args.worker))) + return + if args.work is None or not args.work.is_absolute(): + parser.error("--work must name an external absolute scratch directory") + if args.work.resolve().is_relative_to(ROOT): + parser.error("benchmark outputs must stay outside the checkout") + result = measure(args.work, args.pairs) + output = args.work / "BENCHMARK.json" + with output.open("x") as stream: + json.dump(result, stream, indent=2) + stream.write("\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/research/source-comparison-memory/verify.py b/research/source-comparison-memory/verify.py new file mode 100644 index 0000000000..6afe1766f7 --- /dev/null +++ b/research/source-comparison-memory/verify.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Replay the new source-comparison contracts; do not benchmark in CI. + +Produced by Nick Petalas. Apache-2.0. +""" +import hashlib +import importlib.util +import json +from pathlib import Path +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +def main(): + path = ROOT / "tests/test_research_source_comparison.py" + spec = importlib.util.spec_from_file_location("source_comparison_contracts", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + suite = unittest.defaultTestLoader.loadTestsFromModule(module) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if not result.wasSuccessful() or result.testsRun != 7 or result.skipped: + raise SystemExit("source comparison contracts failed or were skipped") + paths = (ROOT / "scripts/research_package.py", path, Path(__file__).resolve()) + receipt = {"tests": result.testsRun, "failures": len(result.failures), + "errors": len(result.errors), "python": sys.version, + "sha256": {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in paths}} + print("PASS_SOURCE_COMPARISON_MEMORY " + json.dumps(receipt, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/research_package.py b/scripts/research_package.py index 48fe9feb46..d67f655e62 100644 --- a/scripts/research_package.py +++ b/scripts/research_package.py @@ -365,6 +365,28 @@ def source_pin(row, base): + "; accepted pin forms are base_commit: and head:<40-hex sha>:") +def file_equals_bytes(path, expected): + """Compare a stable regular file exactly, with at most a 1 MiB buffer. + + Read the whole initial extent even after a mismatch, propagating I/O errors. + The caller retains responsibility for path validation and checkout stability. + """ + with path.open("rb") as stream: + size = os.fstat(stream.fileno()).st_size + capacity = min(1024 * 1024, max(1, size)) + buffer = bytearray(capacity) + actual = memoryview(buffer) + wanted = memoryview(expected) + matches = size == len(expected) + for offset in range(0, size, capacity): + count = stream.readinto(buffer) + assert 0 <= count <= capacity + if count != min(capacity, size - offset) or actual[:count] != wanted[offset:offset + count]: + matches = False + tail = stream.read(1) + return matches and not tail + + def validate(root, package, check_pins=True): parts = PurePosixPath(package).parts require(len(parts) == 2 and parts[0] == "research" and re.fullmatch(r"[a-z0-9][a-z0-9-]*", parts[1]), @@ -501,7 +523,7 @@ def validate(root, package, check_pins=True): require(obj == row["git_blob"], "source blob provenance mismatch: " + name) require(hashlib.sha256(pinned).hexdigest() == row["sha256"], "source SHA256 provenance mismatch: " + name) if row["usage"] == "input": - require(pinned == relative_file(root, name).read_bytes(), "working source drift: " + name) + require(file_equals_bytes(relative_file(root, name), pinned), "working source drift: " + name) if name == catalog_name: catalog = json.loads(pinned) cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task["task_id"]] diff --git a/tests/test_research_source_comparison.py b/tests/test_research_source_comparison.py new file mode 100644 index 0000000000..f9d2455a2e --- /dev/null +++ b/tests/test_research_source_comparison.py @@ -0,0 +1,144 @@ +"""Exact source validation with bounded working-file comparison. + +Produced by Nick Petalas. Apache-2.0. +""" +import errno +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import new_research_package as scaffold +import research_package as package +sys.path.pop(0) + +BLOCK = 1024 * 1024 + + +class FileComparisonTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.path = Path(self.temp.name) / "binary source\nλ.bin" + + def test_equal_binary_files_across_buffer_boundaries(self): + for size in (0, 1, BLOCK - 1, BLOCK, BLOCK + 1, 2 * BLOCK + 19): + with self.subTest(size=size): + expected = (bytes(range(256)) * ((size + 255) // 256))[:size] + self.path.write_bytes(expected) + self.assertTrue(package.file_equals_bytes(self.path, expected)) + self.assertEqual(self.path.read_bytes(), expected) + + def test_every_boundary_mutation_rejects(self): + expected = bytes(range(256)) * (2 * BLOCK // 256 + 1) + for position in (0, BLOCK - 1, BLOCK, BLOCK + 1, 2 * BLOCK - 1, + 2 * BLOCK, len(expected) - 1): + with self.subTest(position=position): + changed = bytearray(expected) + changed[position] ^= 1 + self.path.write_bytes(changed) + self.assertFalse(package.file_equals_bytes(self.path, expected)) + + def test_empty_truncated_and_appended_sources_reject(self): + for expected, actual in ((b"", b"\0"), (b"x", b""), (b"abc", b"ab"), + (b"abc", b"abcd"), (b"a" * BLOCK, b"a" * BLOCK + b"z")): + with self.subTest(lengths=(len(expected), len(actual))): + self.path.write_bytes(actual) + self.assertFalse(package.file_equals_bytes(self.path, expected)) + + def test_open_error_propagates(self): + with self.assertRaises(FileNotFoundError): + package.file_equals_bytes(self.path, b"") + + def test_read_error_after_mismatch_propagates_and_closes(self): + self.path.write_bytes(b"x" * (BLOCK + 1)) + stream = self.path.open("rb") + reads = 0 + + def readinto(buffer): + nonlocal reads + reads += 1 + if reads == 2: + raise OSError(errno.EIO, "injected read failure") + return stream.readinto(buffer) + + wrapped = mock.MagicMock() + wrapped.__enter__.return_value = wrapped + wrapped.__exit__.side_effect = lambda *args: stream.close() + wrapped.fileno.return_value = stream.fileno() + wrapped.readinto.side_effect = readinto + with mock.patch.object(Path, "open", return_value=wrapped): + with self.assertRaises(OSError) as caught: + package.file_equals_bytes(self.path, b"y" * (BLOCK + 1)) + self.assertEqual(caught.exception.errno, errno.EIO) + self.assertTrue(stream.closed) + + +class SourceValidationTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + subprocess.run(["git", "init", "-q", str(self.root)], check=True, timeout=30) + for key, value in (("user.name", "Fixture"), ("user.email", "fixture@example.invalid"), + ("core.autocrlf", "false")): + package.git(self.root, "config", key, value) + catalog = self.root / "docs/research/agent-task-catalog.json" + catalog.parent.mkdir(parents=True) + catalog.write_text(json.dumps({"priority_revision": 1, "tracks": [{"tasks": [ + {"id": "fixture", "question": "Preserve exact source acceptance"}]}]})) + selected = self.root / "certificates/selected-result.json" + selected.parent.mkdir() + selected.write_text("{}") + self.payload = bytes(range(256)) * (BLOCK // 256 + 1) + self.input = self.root / "source.bin" + self.input.write_bytes(self.payload) + package.git(self.root, "add", ".") + package.git(self.root, "-c", "gc.auto=0", "commit", "-qm", "fixture") + target = scaffold.create(self.root, "fixture", "fixture", "Fixture", ["source.bin"]) + task = json.loads((target / "TASK.json").read_text()) + task.update(outcome="completed", bottleneck_link="Exact source acceptance", + stop_rule="One fixture", evidence="Source drift controls", ai_assistance="Fixture", + open_questions=[], no_open_questions_reason="Bounded fixture only") + (target / "TASK.json").write_text(json.dumps(task)) + manifest = json.loads((target / "VERIFY.json").read_text()) + manifest["status"] = "ready" + manifest["commands"][0]["checks"] = "Source drift fixture" + (target / "VERIFY.json").write_text(json.dumps(manifest)) + self.target = target + + def test_complete_validator_accepts_restored_source_and_rejects_drift(self): + first = package.validate(self.root, "research/fixture") + mutations = (b"", self.payload[:-1], self.payload + b"x", + b"x" + self.payload[1:], self.payload[:-1] + b"x") + for changed in mutations: + self.input.write_bytes(changed) + with self.assertRaisesRegex(ValueError, "working source drift: source.bin"): + package.validate(self.root, "research/fixture") + self.input.write_bytes(self.payload) + self.assertEqual(package.validate(self.root, "research/fixture"), first) + self.assertEqual(hashlib.sha256(self.input.read_bytes()).digest(), + hashlib.sha256(self.payload).digest()) + + def test_complete_validator_retains_blob_and_sha_rejections(self): + path = self.target / "SOURCES.json" + original = json.loads(path.read_text()) + for field, replacement in (("git_blob", "0" * 40), ("sha256", "0" * 64)): + changed = json.loads(json.dumps(original)) + row = next(r for r in changed["files"] if r["path"] == "source.bin") + row[field] = replacement + path.write_text(json.dumps(changed)) + with self.assertRaisesRegex(ValueError, "provenance mismatch: source.bin"): + package.validate(self.root, "research/fixture") + path.write_text(json.dumps(original)) + package.validate(self.root, "research/fixture") + + +if __name__ == "__main__": + unittest.main() From ac7949b52fb71417bc5d48b8a562042b8e78cf89 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Wed, 28 Oct 2026 16:23:24 +0000 Subject: [PATCH 2/2] test(research): use a portable source fixture name --- tests/test_research_source_comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_research_source_comparison.py b/tests/test_research_source_comparison.py index f9d2455a2e..54da9c0af3 100644 --- a/tests/test_research_source_comparison.py +++ b/tests/test_research_source_comparison.py @@ -25,7 +25,7 @@ class FileComparisonTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) - self.path = Path(self.temp.name) / "binary source\nλ.bin" + self.path = Path(self.temp.name) / "binary source λ.bin" def test_equal_binary_files_across_buffer_boundaries(self): for size in (0, 1, BLOCK - 1, BLOCK, BLOCK + 1, 2 * BLOCK + 19):