diff --git a/research/windows-git-stream-ownership/.gitattributes b/research/windows-git-stream-ownership/.gitattributes new file mode 100644 index 0000000000..008992b8f3 --- /dev/null +++ b/research/windows-git-stream-ownership/.gitattributes @@ -0,0 +1,2 @@ +# Preserve the exact executable and receipt bytes validated on Windows and WSL. +* -text whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol diff --git a/research/windows-git-stream-ownership/LICENSE b/research/windows-git-stream-ownership/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/research/windows-git-stream-ownership/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/research/windows-git-stream-ownership/NOTICE b/research/windows-git-stream-ownership/NOTICE new file mode 100644 index 0000000000..5bcd615910 --- /dev/null +++ b/research/windows-git-stream-ownership/NOTICE @@ -0,0 +1,17 @@ +Windows streaming Git ownership experiment + +The unmodified baseline/research_package.py is from CrocSwap/integer-mult-bounds +commit 32890bc5de034ab8fc93ac88ead7279a6a416d04. +Copyright 2026 Douglas Colkitt. Prepared with OpenAI Codex assistance. + +The Job Object lifecycle, native structures and isolated event gate in the +adapter derive from ywz65's PR2469 and its retained successors. Native handle +observation follows ywz65's PR2452; the response deadline and POSIX lifetime +controls follow Brian / JoyboyBrian's PR4738 and PR5185. Nick Petalas's PR2752 +and the credited source reader/fallback lineage remain credited. The README +names the Windows, runner, process-tree and source/export predecessors. + +New stream adapter and bounded composition controls prepared for Mihir +Srivastava with substantial OpenAI Codex assistance. Apache-2.0, as in LICENSE. +This is not an official OpenAI release or endorsement. Tooling checks do not +establish a mathematical result or independent human scientific review. diff --git a/research/windows-git-stream-ownership/README.md b/research/windows-git-stream-ownership/README.md new file mode 100644 index 0000000000..fdaac3e593 --- /dev/null +++ b/research/windows-git-stream-ownership/README.md @@ -0,0 +1,180 @@ +# Opt-in Windows ownership for streaming Git source reads + +This package supplies a **tested streaming Job Object adapter**, integrated +with the exact Git blob reader from canonical main +`32890bc5de034ab8fc93ac88ead7279a6a416d04`. It fixes the observed lifetime +failures **in this opt-in composition**. It does not install a change in the +shared runner, repin an earlier package, or change the selected frontier. + +Both normal and optimized native registered runs pass all 25 process scenarios, +the assignment-denial control, and real-Git exact binary/duplicate reads with +and without the adapter. The registered launcher supplies an enclosing Job +Object, so these runs also exercise nested ownership. See the +[normal](evidence/windows-normal.json), [optimized](evidence/windows-optimized.json) +and [portable Linux](evidence/linux-portable.json) receipts. These are +author-checked tooling results, not independent human review or mathematical +acceptance. + +## Concrete delta + +The frozen reader has a direct-child Windows `kill()`. Once that child exits, +a live descendant can retain its stdin/stdout handles. Killing the exited leader +does not release the pipe, so the response watchdog fires while the reader stays +blocked. Normal completion, malformed replies, early close and the legacy +fallback can also leave descendants running. + +`owned_pipe.OwnedPopen` exposes the subset of Popen used by that reader: + +1. Create a private kill-on-close job and a closed event gate. +2. Launch the existing isolated `-I -S` gate, passing stdin/stdout handles + directly to the requested command. +3. Assign the gate to the job before releasing it to spawn the command. +4. Make `kill()` terminate the job and wait for zero active processes. A lock + makes this stop idempotent when used by the watchdog and cleanup. +5. On context exit, stop the job, reap the gate, and close pipe/event/job handles. + Assignment failure never releases the gate. + +The native structures, startup isolation, Unicode JSON argv handling and +pre-spawn gate come from ywz65's existing `run_windows_command`, with retained +successors. The new part is its stream lifetime interface and its composition +with the unchanged reader. There is no Python-level pipe relay or blob buffering +added by the adapter; the original one-blob streaming, validation and fallback +code executes from the authenticated frozen bytes. + +Microsoft documents default CreateProcess descendant membership and +kill-on-last-handle-close in its +[Job Objects contract](https://learn.microsoft.com/en-us/windows/win32/procthread/job-objects); +[TerminateJobObject](https://learn.microsoft.com/en-us/windows/win32/api/jobapi2/nf-jobapi2-terminatejobobject) +terminates current members and nested jobs. These OS contracts motivate the +implementation. The finite tests validate the recorded native environment, +not every Windows host policy or process-creation mechanism. + +## Source boundary and reproduction + +Python 3.12 and Git; standard library only. From the repository root: + +```text +python -I -W error -B research/windows-git-stream-ownership/verify.py --require-windows +python -I -W error -B -O research/windows-git-stream-ownership/verify.py --require-windows +python -B scripts/research_package.py --package research/windows-git-stream-ownership +python -B scripts/check_research_commit.py --package research/windows-git-stream-ownership +``` + +A Windows run prints `PASS_WINDOWS_GIT_STREAM_OWNERSHIP`. On another OS the +default command runs six portable protocol/deadline controls plus real Git, +and prints **`PASS_PORTABLE_CONTROLS_WINDOWS_NOT_RUN`** with +`windows_observations_executed: false`. Passing Linux CI does not run or +certify the Windows adapter. `--require-windows` rejects that reduced coverage. +The registered commands run both normal and optimized Python; optimization +propagates to the fixture workers, and checks use explicit exceptions. + +The direct verifier also runs in a Git-free copy of this package. It needs an +installed Git executable only for fresh temporary object repositories. +`baseline/research_package.py` is a byte-identical behavioral control, not a +modified predecessor. Its SHA256 is +`84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6`, +bound to Git blob `48fe9feb4699c09e95505d95c327ea998ac8b3a7` in +`SOURCES.json`. The verifier compiles the authenticated bytes directly rather +than loading a preexisting bytecode cache. The copied reader is needed to keep +the before/after experiment reproducible after a future production repair. + +All source rows are historical provenance. The package executes its local +frozen reader and adapter, so there are no live external inputs to repin when +main advances. The repository metadata checker additionally authenticates those +provenance rows. Receipt hashes bind the two new implementation files. + +## Evidence and distinguishing controls + +The fixture substitutes a controlled Python producer for the Git process at +the Popen boundary. Its protocol and OS pipes are real; it does not claim that +an installed Git executable spontaneously produces the stall. + +Before releasing the producer, the observer waits for readiness and retains +native handles to the actual producer, its child, and the adapter's gate where +present. Later PID reuse cannot count as termination. These handles are observed +before any fixture cleanup. Only fixture processes are stopped in teardown. + +| Cases | Frozen reader | Adapter composition | +|---|---|---| +| Header, partial payload, absent terminator, blocked write | Producer has exited; child alive; watchdog fired; reader still pending 1.2 seconds later. Terminating that child then releases the pipe and returns TimeoutExpired. | TimeoutExpired returns and all retained process handles stop before teardown. | +| Exact binary success, malformed header, early generator close, first-request exit-129 fallback | Reader returns the expected bytes/error, but child remains alive. | Same bytes/error/fallback count; descendants and gate stop before teardown. | +| Consumer pause of 1.2 seconds between replies | Direct-child positive control passes. | Two exact replies pass despite the pause exceeding the 0.8-second response deadline; no abandoned descendant remains. | +| Assignment denied with native error 5 | Not applicable. | Error retained, unassigned gate reaped, streams closed, requested-command marker absent. | + +Six portable cases are no-descendant header/payload/terminator stalls, +binary success, malformed reply and consumer suspension. Eight Windows cases +reproduce the old failure boundaries. Two separate whole-command controls use +the unchanged registered launcher: it stops the stalled experiment at its +15-second outer timeout and cleans descendants after success. Those results +are not credited to the blob reader. Nine Windows cases run the streaming +adapter. The real Git control checks exact NUL/non-UTF-8/CR/LF bytes and duplicate +requests both with the baseline and with the adapter. + +The response timer is shortened from 90 seconds to 0.8 seconds only in the +temporary worker. Startup readiness happens before that timer starts. Whole-job +cleanup retains the predecessor's 10-second accounting limit; harness waits are +separate failure bounds. The early-close case retains the frozen reader's +existing five-second leader grace period. This is not a tight latency benchmark. + +Native receipts use Windows 11 build 26300 / CPython 3.12.0; the portable receipt +uses Ubuntu/WSL2 / CPython 3.12.3. An initial five-second whole-command control +exited before readiness; the final harness uses explicit absolute script paths +and a 15-second outer budget. The reader's 0.8-second response budget is unchanged. +A disposable baseline-byte mutation rejects before fixture launch, and Linux +`--require-windows` rejects rather than misreporting native coverage. + +## Scope, overlap and attribution + +The `portable-replay` card is changed from canonical payload comparison to +the explicit Windows source-reader lifetime handoff in +[Discussion #4681](https://github.com/CrocSwap/integer-mult-bounds/discussions/4681#discussioncomment-19152845). +The [canonical scoped claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/4681#discussioncomment-19170460) +records the correction: the full overlap refresh surfaced two earlier concurrent +audit claims, +[19170424](https://github.com/CrocSwap/integer-mult-bounds/discussions/4681#discussioncomment-19170424) +and [19170434](https://github.com/CrocSwap/integer-mult-bounds/discussions/4681#discussioncomment-19170434). +This work released the duplicate audit scope and supplies a complementary +ownership adapter; its baseline observations are controls, not a novelty claim. + +The entry refresh covered 331 open PR bodies, 4,728 Discussion records and all +7,402 tracked in-scope README/TASK/ledger notes, with complete reported +pagination and inventory. It excludes arbitrary historical diffs and unpublished +work. The selected metadata-only baseline passed at priority revision 7, +selected #5413; the complete selected cold replay was not run. + +The submission refresh still uses the same canonical main. It covered 332 +open PRs at 22:46 UTC, 4,732 Discussion records at 22:48 UTC and all 7,402 +tracked notes (plus this package's two untracked notes). The concurrent +return-lifetime audit is now [PR #5461](https://github.com/CrocSwap/integer-mult-bounds/pull/5461); +the other baseline probe is published at +[7a9a2b67e1967ba31b7330031daae186d6b87a86](https://github.com/mihir-s-05/integer-mult-bounds/tree/7a9a2b67e1967ba31b7330031daae186d6b87a86/docs/research/windows-git-pipe-audit). +Their diagnosis is complementary evidence, not an implementation dependency. + +Credit **ywz65 / #2469** for the gated Job Object lifecycle and **#2452** for +retained process-handle observation; **Brian / JoyboyBrian / #4738 and #5185** +for the response deadline, POSIX descendant fixture and source-reader lifetime +contract; **Nick Petalas / #2752** for the fallback and source-reader lineage; +**Mihir Srivastava / #4686, #5358 and #5047** for retained Windows pipe, +installed-Git and Unicode argument work; **Douglas Colkitt** for the package +runner; **Paul Romeo, yssite, Arham Tahir, Woklerx and Tekalign** for retained +lifecycle/source/export predecessors. The original source header is unchanged. + +Prepared for Mihir Srivastava with substantial OpenAI Codex assistance, one +agent. Apache-2.0; [LICENSE](LICENSE) and [NOTICE](NOTICE) apply. No new kappa, supplier +admission, unconditional theorem or independent human/formal review is claimed. + +## Open next questions + +1. **Production adoption.** Review whether to factor the shared command launcher + into a reusable lifetime context and use it in `_git_blob_batch`. Start with + this Popen-subset contract and the independent concurrent audit fixtures. + Updating production test seams, source-pin invalidation and all affected + consumers remains a separate review task. Spawn-error diagnostic equivalence + is not established: this gate changes the process-launch boundary. This is + a Popen subset for the frozen reader, not a general subprocess replacement. +2. **Actual deployment.** Bind the adapter to an identified real Git/helper + invocation under the intended enclosing Job Object policy. Controlled + producers establish the pipe-boundary failure and repair, not every helper + chain. Alternate Windows/Python versions, restrictive host policies, owner + process death during setup, non-CreateProcess descendants and hostile + breakaway attempts were not tested. Assignment failure is explicitly covered. diff --git a/research/windows-git-stream-ownership/SOURCES.json b/research/windows-git-stream-ownership/SOURCES.json new file mode 100644 index 0000000000..f9477d71d1 --- /dev/null +++ b/research/windows-git-stream-ownership/SOURCES.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "base_commit": "32890bc5de034ab8fc93ac88ead7279a6a416d04", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "fc3d4ba105e1f097d0d7d051a308281e86ab1446", + "sha256": "66974efdced4083c00025f97d95d83909a226cdf801edb9301c195c6641f6903" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "f126dd2e2dad2b669e75d1d0d0bd05c600725602", + "sha256": "f7a48bfa2659b88378282a8eee7d9fe52cdf505594124c4d1686b58510fffd72" + }, + { + "path": "scripts/research_package.py", + "usage": "provenance", + "git_blob": "48fe9feb4699c09e95505d95c327ea998ac8b3a7", + "sha256": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6" + }, + { + "path": "tests/test_research_git_descendants.py", + "usage": "provenance", + "git_blob": "599bbd707dfda57cea562a002195ea658e2f03bc", + "sha256": "ce212173528b31e961f82295c6795e6674b9062d215c30d3d0e148bb4c41f874" + }, + { + "path": "tests/test_research_windows_job.py", + "usage": "provenance", + "git_blob": "ddd4c5ab0061a2069aae436590d6b5f1d5ceb461", + "sha256": "63577aea850d79fb9af2b87f5dce225eeaa34736f2115a6f4061db75f57e7877" + }, + { + "path": "LICENSE", + "usage": "provenance", + "git_blob": "261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + } + ] +} diff --git a/research/windows-git-stream-ownership/TASK.json b/research/windows-git-stream-ownership/TASK.json new file mode 100644 index 0000000000..960eb4f124 --- /dev/null +++ b/research/windows-git-stream-ownership/TASK.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "task_id": "portable-replay", + "priority_revision": 7, + "base_commit": "32890bc5de034ab8fc93ac88ead7279a6a416d04", + "original_card": { + "id": "portable-replay", + "question": "Can one reproducible environment or artifact-comparison failure be repaired?", + "starting_points": [ + "research/gzip-container-reproducibility/README.md", + "docs/ci-verification.md" + ], + "methods": [ + "Reproduce one portability failure and add a minimal cross-environment fix.", + "Audit one canonical-payload comparison for omitted proof-relevant fields." + ], + "acceptance_gate": "Preserve immutable source hashes and proof-relevant content; a different compressed byte stream alone is not a mathematical mismatch.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Read the gzip-container reproducibility examples and distinguish frozen source bytes from regenerated payloads.", + "scope": "Positive control / source-bound orientation only; this is not a new construction or full cold replay." + }, + "first_experiment": "Reproduce one specific mismatch or audit one canonical comparison for omitted proof-relevant fields.", + "bottleneck_link": "Portable exact checks prevent environment failures from blocking valid contributions.", + "allowed_variations": [ + "One interpreter/version mismatch or one payload field comparison." + ], + "scope_change_triggers": [ + "Removing immutable-source pins because generated compressed bytes differ." + ], + "overlap_terms": [ + "gzip container", + "reproducibility", + "canonical payload" + ] + }, + "scope_status": "changed", + "actual_scope": "Opt-in Windows Job Object stream adapter integrated with the authenticated frozen Git blob reader", + "scope_change_reason": "Take the explicit Windows descendant handoff. Concurrent audit claims 19170424/19170434 appeared during overlap refresh, so release the duplicate audit and supply the complementary streaming ownership adapter. The retained baseline observations are failure controls, not a novelty claim.", + "bottleneck_link": "A blocked immutable-source read prevents replay; the adapter tests lifetime ownership while retaining exact source bytes and fallback semantics.", + "budget_minutes": 60, + "stop_rule": "Stop after the bounded actual-reader integration and native process controls; production integration, other Git callers and a shared-launcher refactor remain separate.", + "outcome": "partial", + "evidence": "README.md; evidence/windows-normal.json; evidence/windows-optimized.json", + "author": "Mihir Srivastava, with substantial OpenAI Codex assistance", + "ai_assistance": "Substantial OpenAI Codex assistance, one agent; no independent human review claimed", + "open_questions": [ + { + "question": "Should the shared reader adopt this streaming owner through a reusable launcher lifecycle?", + "why_it_matters": "The live production reader still has the Windows lifetime gap; an opt-in source-bound adapter does not change it.", + "first_step": "Review owned_pipe.py and adapt the shared runner with the same source-protocol and native ownership controls, coordinating with the two baseline-audit owners.", + "blocked_by": "Production integration, all affected test seams and pinned-source invalidation review are not covered here." + }, + { + "question": "Do the ownership contract and gate work with the intended deployed Git helper chain and host Job Object policy?", + "why_it_matters": "Controlled Python producers reproduce the pipe boundary but do not establish behavior of every Git/helper deployment.", + "first_step": "Run the adapter on an identified actual helper invocation under the intended host job policy with retained handles.", + "blocked_by": "No naturally stalled Git/helper incident or alternate native Windows environment was supplied." + } + ], + "overlap": { + "canonical_thread": "https://github.com/CrocSwap/integer-mult-bounds/discussions/4681#discussioncomment-19170460", + "open_prs": 332, + "discussion_records": 4732, + "tracked_notes": 7402, + "complete_reported_coverage": true, + "limitation": "Body/note search, not historical diffs or unpublished work", + "checked_at": "2026-10-24T22:48:46Z", + "concurrent_audit_pr": 5461 + } +} diff --git a/research/windows-git-stream-ownership/VERIFY.json b/research/windows-git-stream-ownership/VERIFY.json new file mode 100644 index 0000000000..3b100f6872 --- /dev/null +++ b/research/windows-git-stream-ownership/VERIFY.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 240, + "checks": "Frozen-reader negative controls; opt-in Windows streaming owner and setup rejection; explicit portable-only status elsewhere" + }, + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "-O", + "{package}/verify.py" + ], + "timeout_seconds": 240, + "checks": "Frozen-reader negative controls; opt-in Windows streaming owner and setup rejection; explicit portable-only status elsewhere" + } + ] +} diff --git a/research/windows-git-stream-ownership/baseline/research_package.py b/research/windows-git-stream-ownership/baseline/research_package.py new file mode 100644 index 0000000000..48fe9feb46 --- /dev/null +++ b/research/windows-git-stream-ownership/baseline/research_package.py @@ -0,0 +1,1026 @@ +#!/usr/bin/env python3 +"""Validate and replay explicitly registered research packages, never every verify.py. + +Douglas Colkitt, with OpenAI Codex assistance. Apache-2.0. +""" +import argparse +import errno +from contextlib import closing, contextmanager +from itertools import chain +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path, PurePosixPath +import re +import signal +import stat +import subprocess +import sys +import tempfile +import threading + +ROOT = Path(__file__).resolve().parents[1] +GIT_BLOB_RESPONSE_TIMEOUT = 90 + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def canonical_relative_path(name): + # Git's changed paths use one repository-relative POSIX spelling. Accepting + # aliases here would let a declared input evade the planner's exact match. + require(isinstance(name, str), "canonical relative path must be a string") + path = PurePosixPath(name) + require(bool(path.parts) and not path.is_absolute() and ".." not in path.parts + and "\\" not in name and path.as_posix() == name, + "use a canonical repository-relative POSIX path: " + name) + return path + + +def symlink_free_relative_path(root, name): + """Reject symlink/junction components without requiring a deleted input to exist. + + Git reports changes under the target's path, not a symlink alias declared + by its consumer. Inspect existing components below the repository root; + the checkout must remain stable during planning, validation and replay. + """ + parts = canonical_relative_path(name).parts + current = root + for part in parts: + current = current / part + try: + info = current.lstat() + except (FileNotFoundError, NotADirectoryError): + break + require(not stat.S_ISLNK(info.st_mode), "symlink in repository-relative path: " + name) + # Windows junctions have directory mode, not S_IFLNK. Inspect the same + # lstat result so even a dangling junction is rejected before traversal. + require(getattr(info, "st_reparse_tag", 0) != getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", None), + "junction in repository-relative path: " + name) + return root / name + + +def relative_file(root, name): + result = symlink_free_relative_path(root, name) + require(result.resolve().is_relative_to(root.resolve()) and result.is_file(), + "missing or external file: " + name) + return result + + +# Adapted from the package-local policy in +# research/t74-2530-source-partner-audit/audit_common.py (Apache-2.0). +# Keep repository configuration and linked-worktree .git discovery intact; +# inherited shell overrides must not choose a different repository or index. +_GIT_REDIRECT_VARS = frozenset(( + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE", "GIT_CEILING_DIRECTORIES", + "GIT_COMMON_DIR", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", +)) +_GIT_CONFIG_PREFIXES = ("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_") + + +def git_source_env(): + """Use cwd's repository and stored objects, without changing the parent. + + Git can ignore cwd when inherited selection variables point elsewhere. + Config-injection variables can also redirect worktree/index operations. + This policy is only for Git metadata/source subprocesses, not verifiers. + """ + env = {name: value for name, value in os.environ.items() + if name not in _GIT_REDIRECT_VARS and not name.startswith(_GIT_CONFIG_PREFIXES)} + # Replacement refs can report an original ID while returning other bytes. + env["GIT_NO_REPLACE_OBJECTS"] = "1" + return env + + +def git(root, *args): + return subprocess.run(["git", *args], cwd=root, env=git_source_env(), + capture_output=True, check=True).stdout + + +def tracked_input_paths(root): + """Use HEAD spelling, never case folding, filesystem resolution or the index. + + Staged/untracked files are not replay inputs until committed. NUL framing + preserves Git filenames containing whitespace, including CR and LF. + """ + raw = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", "-z", "HEAD"], cwd=root, env=git_source_env(), + capture_output=True, check=True, timeout=30).stdout + return frozenset(name.decode("utf-8") for name in raw.split(b"\0") if name) + + +def require_tracked_input(name, paths): + canonical_relative_path(name) + require(name in paths, "use an exact Git-tracked input path from HEAD: " + name) + + +def git_source_rows(root, base, names): + """Read one pinned object at a time through one Git process; no persistent cache. + + Each request is consumed before the next is sent. This avoids pipe deadlocks + and retains the old one-blob memory bound, even for large source archives. + Newline paths and non-blob objects retain the legacy Git-show semantics. + The caller must close this generator if source validation fails mid-stream. + """ + with subprocess.Popen(["git", "cat-file", "--batch"], cwd=root, env=git_source_env(), + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as process: + try: + for name in names: + expression = base + ":" + name + if "\n" in expression or "\r" in expression: + yield git(root, "rev-parse", expression).decode().strip(), git(root, "show", expression) + continue + process.stdin.write(expression.encode("utf-8") + b"\n") + process.stdin.flush() + header = process.stdout.readline().rstrip(b"\n").split(b" ") + require(len(header) == 3 and re.fullmatch(rb"[0-9a-f]{40,64}", header[0]) + and header[2].isdigit(), "missing or invalid Git source: " + name) + size = int(header[2]) + payload = process.stdout.read(size) + require(len(payload) == size and process.stdout.read(1) == b"\n", + "truncated Git source: " + name) + if header[1] != b"blob": + payload = git(root, "show", expression) + yield header[0].decode("ascii"), payload + process.stdin.close() + code = process.wait(timeout=90) + require(code == 0, "Git source reader failed: " + process.stderr.read().decode("utf-8", errors="replace")) + finally: + if not process.stdin.closed: + process.stdin.close() + if process.poll() is None: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +class _UnsupportedNulBatch(Exception): + """Git rejected the initial NUL-mode request before yielding any source.""" + + +def _closed_stdin_error(error): + # Windows can report EINVAL instead of EPIPE when a child closes stdin. + # This classifies the pipe error only; the retry still requires exit 129. + return (isinstance(error, BrokenPipeError) + or (os.name == "nt" and error.errno == errno.EINVAL)) + + +def _stop_git_blob_process(process): + """Stop this reader's owned POSIX group even after its leader exits.""" + try: + if os.name == "posix": + # Reap an exited leader first: signaling a group containing only + # its unreaped zombie can report EPERM on macOS. A surviving + # descendant keeps the group alive after the leader is reaped. + process.poll() + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except ProcessLookupError: + pass + + +@contextmanager +def _git_blob_response_deadline(stop): + """Bound one response, excluding time spent by its consumer. + + On POSIX the reader owns a session: ordinary descendants retaining a pipe + are stopped with the leader. Escaped sessions and separate git() calls are + outside this contract; non-POSIX retains the direct-child deadline. + """ + expired = threading.Event() + + def terminate(): + expired.set() + stop() + + timer = threading.Timer(GIT_BLOB_RESPONSE_TIMEOUT, terminate) + timer.start() + try: + yield + finally: + timer.cancel() + timer.join() + if expired.is_set(): + raise subprocess.TimeoutExpired("git cat-file blob response", GIT_BLOB_RESPONSE_TIMEOUT) + + +def _git_blob_batch(root, specs, nul): + argv = ["git", "cat-file", "--batch"] + (["-z"] if nul else []) + # Usage diagnostics from the unsupported-option probe are expected. Avoid + # a second pipe that could block while we consume the framed stdout reply. + with subprocess.Popen(argv, cwd=root, env=git_source_env(), stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + start_new_session=(os.name == "posix")) as process: + stopped = False + + def stop(): + nonlocal stopped + if not stopped: + _stop_git_blob_process(process) + stopped = True + + try: + for index, spec in enumerate(specs): + require("\0" not in spec, "invalid Git object request") + request = spec + if not nul and ("\n" in spec or "\r" in spec): + # Resolve one literal argument, never split a path into requests. + request = git(root, "rev-parse", "--verify", "--end-of-options", spec).decode("ascii").strip() + require(re.fullmatch(r"[0-9a-f]{40,64}", request), "invalid Git object ID") + with _git_blob_response_deadline(stop): + try: + process.stdin.write(request.encode("utf-8") + (b"\0" if nul else b"\n")) + process.stdin.flush() + except OSError as exc: + if not _closed_stdin_error(exc): + raise + header = b"" + else: + header = process.stdout.readline() + if nul and index == 0 and not header and process.wait(timeout=90) == 129: + raise _UnsupportedNulBatch(spec) + fields = header.rstrip(b"\n").split(b" ") + require(len(fields) == 3 and re.fullmatch(rb"[0-9a-f]{40,64}", fields[0]) + and fields[1] == b"blob" and fields[2].isdigit(), + "missing or non-blob Git source: " + spec) + size = int(fields[2]) + pinned = process.stdout.read(size) + require(len(pinned) == size and process.stdout.read(1) == b"\n", + "incomplete Git blob response: " + spec) + yield fields[0].decode("ascii"), pinned + process.stdin.close() + require(process.wait(timeout=90) == 0, "Git source reader failed") + finally: + # Terminate ordinary descendants before closing buffered pipes. This + # also covers successful leader exit and an early generator close. + if os.name == "posix": + stop() + # Option rejection can close stdin before our first write/flush. + try: + process.stdin.close() + except OSError as exc: + if not _closed_stdin_error(exc): + raise + process.stdout.close() + if process.poll() is None: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def git_blobs(root, specs): + """Stream exact blobs, retaining the one-process NUL protocol on newer Git. + + Git without -z rejects it with usage status 129. Retry only before any + response, replaying the consumed first request through the line protocol. + The fallback resolves CR/LF paths to object IDs before requesting content. + The caller must close this iterator if validation stops before exhaustion. + """ + specs = iter(specs) + sentinel = object() + first = next(specs, sentinel) + if first is sentinel: + return + with closing(_git_blob_batch(root, chain((first,), specs), nul=True)) as rows: + try: + yield from rows + except _UnsupportedNulBatch as exc: + with closing(_git_blob_batch(root, chain((exc.args[0],), specs), nul=False)) as legacy: + yield from legacy + + +def meaningful(value): + return isinstance(value, str) and bool(value.strip()) and not re.search(r"\bTODO\b", value, re.I) + + +def brief(value): + text = json.dumps(value, ensure_ascii=True, sort_keys=True) + return text if len(text) <= 120 else text[:117] + "..." + + +def require_key(mapping, key, where, shape=None): + """Name the missing key and its expected shape instead of a bare KeyError.""" + require(isinstance(mapping, dict), + where + " must be a JSON object; found " + type(mapping).__name__ + ": " + brief(mapping)) + require(key in mapping, where + "." + key + " is missing" + + ("; expected " + shape if shape else "")) + return mapping[key] + + +def require_text(mapping, key, where, shape="a non-empty string without TODO"): + """Name the key and distinguish missing, wrong-shaped and weak values.""" + require(isinstance(mapping, dict), + where + " must be a JSON object; found " + type(mapping).__name__ + ": " + brief(mapping)) + require(key in mapping, where + "." + key + " is missing; expected " + shape) + value = mapping[key] + require(isinstance(value, str) and meaningful(value), + where + "." + key + " must be " + shape + "; found " + type(value).__name__ + ": " + brief(value)) + + +def card_differences(own, pinned): + """Name every key by which a TASK.json original_card differs from its pin.""" + if not isinstance(own, dict): + return "TASK.json original_card is " + type(own).__name__ + ", expected a JSON object" + details = [] + missing = [key for key in pinned if key not in own] + extra = [key for key in own if key not in pinned] + if missing: + details.append("missing keys: " + ", ".join(missing)) + if extra: + details.append("unexpected keys: " + ", ".join(extra)) + for key in (key for key in pinned if key in own and own[key] != pinned[key]): + details.append(key + " differs (TASK.json: " + brief(own[key]) + "; pinned: " + brief(pinned[key]) + ")") + return "; ".join(details) or "cards compare unequal without a key difference" + + +def source_pin(row, base): + """Resolve one SOURCES.json row to its exact Git object expression. + + Rows without an explicit pin keep the historical base_commit: form. + A file vendored from a pull-request head is pinned exactly with + head:<40-hex sha>:, so its bytes resolve at that commit, not at the + package base. An explicit but unparseable pin fails closed. + """ + name = row["path"] + pin = row.get("pin") + if pin is None: + return base + ":" + name + if pin == "base_commit:" + name: + return base + ":" + name + # Git filenames can contain LF; it is part of the exact path, not a delimiter. + match = re.fullmatch(r"head:([0-9a-f]{40}):(.+)", pin, re.DOTALL) if isinstance(pin, str) else None + if match is not None and match.group(2) == name: + return match.group(1) + ":" + name + raise ValueError("SOURCES.json pin for " + name + " is not a supported pin form: " + brief(pin) + + "; accepted pin forms are base_commit: and head:<40-hex sha>:") + + +def validate(root, package, check_pins=True): + parts = PurePosixPath(package).parts + require(len(parts) == 2 and parts[0] == "research" and re.fullmatch(r"[a-z0-9][a-z0-9-]*", parts[1]), + "package must be research/") + data = {} + for name in ("TASK.json", "SOURCES.json", "VERIFY.json"): + data[name] = json.loads(relative_file(root, package + "/" + name).read_text(encoding="utf-8")) + version = require_key(data[name], "schema_version", name) + require(version == 1, name + ".schema_version must be 1; found " + brief(version)) + task, sources, manifest = (data[n] for n in ("TASK.json", "SOURCES.json", "VERIFY.json")) + scope_status = require_key(task, "scope_status", "TASK.json", "unchanged, refined or changed") + require(scope_status in ("unchanged", "refined", "changed"), + "TASK.json.scope_status must be unchanged, refined or changed; found " + brief(scope_status)) + for field in ("actual_scope", "bottleneck_link", "stop_rule", "evidence", "author", "ai_assistance"): + require_text(task, field, "TASK.json") + if scope_status == "unchanged": + question = require_key(require_key(task, "original_card", "TASK.json", + "the pinned catalogue card object"), + "question", "TASK.json.original_card") + require(task["actual_scope"] == question, + "different wording/scope requires refined or changed plus a reason" + + " (TASK.json.actual_scope " + brief(task["actual_scope"]) + + " differs from TASK.json.original_card.question " + brief(question) + ")") + else: + require_text(task, "scope_change_reason", "TASK.json", + "a non-empty string without TODO explaining the scope refinement/change") + outcome = require_key(task, "outcome", "TASK.json", + "partial, completed, blocked or negative-result") + require(outcome in ("partial", "completed", "blocked", "negative-result"), + "TASK.json.outcome must be partial, completed, blocked or negative-result; found " + brief(outcome)) + questions = task.get("open_questions") + require(isinstance(questions, list), + "TASK.json.open_questions must be a list; found " + type(questions).__name__ + ": " + brief(questions)) + if not questions: + require_text(task, "no_open_questions_reason", "TASK.json", + "a non-empty string without TODO explaining an empty next-question list") + for position, question in enumerate(questions): + for field in ("question", "why_it_matters", "first_step", "blocked_by"): + require_text(question, field, "TASK.json.open_questions[" + str(position) + "]") + readme = relative_file(root, package + "/README.md").read_text(encoding="utf-8") + require("## Open next questions" in readme, "README.md needs an \"## Open next questions\" section") + status = require_key(manifest, "status", "VERIFY.json") + require(status == "ready", + "VERIFY.json.status must be \"ready\"; complete the draft before requesting replay; found " + brief(status)) + python = require_key(manifest, "python", "VERIFY.json") + require(python == "3.12", + "VERIFY.json.python must be \"3.12\" (generic replay currently uses Python 3.12); found " + brief(python)) + requirements = manifest.get("requirements", []) + require(isinstance(requirements, list), + "VERIFY.json.requirements must be a list of exact \"name==version\" pins; found " + + type(requirements).__name__ + ": " + brief(requirements)) + require(len(requirements) == len(set(requirements)), "VERIFY.json.requirements must not repeat a pin") + for item in requirements: + require(isinstance(item, str) + and re.fullmatch(r"(?:numpy|sympy|scipy)==[0-9]+(?:\.[0-9]+){1,2}", item), + "generic CI supports exact numpy/sympy/scipy pins; use a dedicated workflow for other environments" + + "; offending entry: " + brief(item)) + names = [item.split("==")[0] for item in requirements] + require(len(names) == len(set(names)), "declare only one version per dependency") + # Validation already treats an omitted list as empty; execution must receive + # that same default without rewriting the contributor's manifest on disk. + manifest["requirements"] = requirements + commands = manifest.get("commands") + require(isinstance(commands, list) and 1 <= len(commands) <= 8, + "VERIFY.json.commands must be a list of 1..8 explicit verification commands; found " + + type(commands).__name__ + ": " + brief(commands)) + total = 0 + for position, command in enumerate(commands): + path = "VERIFY.json.commands[" + str(position) + "]" + require(isinstance(command, dict), + path + " must be a JSON object; found " + type(command).__name__ + ": " + brief(command)) + argv = command.get("argv") + require(isinstance(argv, list) and len(argv) >= 2 and all(isinstance(a, str) for a in argv), + path + ".argv must be a list of at least 2 strings; found " + brief(argv)) + require(argv[0] == "{python}", path + ".argv must explicitly use {python}; no implicit shell") + require_text(command, "checks", path, "a non-empty string without TODO describing command coverage") + seconds = command.get("timeout_seconds") + require(type(seconds) is int, + path + ".timeout_seconds must be an integer number of seconds; found " + + type(seconds).__name__ + ": " + brief(seconds)) + require(1 <= seconds <= 600, + path + ".timeout_seconds " + str(seconds) + + (" exceeds the 600 s per-command cap" if seconds > 600 + else " is below the 1 s per-command minimum")) + total += seconds + require(total <= 1200, "declared replay budget " + str(total) + + " s (sum of timeout_seconds) exceeds the 1200 s total cap" + + "; use a dedicated job for larger checks") + inputs = manifest.get("inputs") + require(isinstance(inputs, list), + "VERIFY.json.inputs must be a list of canonical repository-relative paths; found " + + type(inputs).__name__ + ": " + brief(inputs)) + tracked = tracked_input_paths(root) + for name in inputs: + symlink_free_relative_path(root, name) + require_tracked_input(name, tracked) + relative_file(root, name) + files = require_key(sources, "files", "SOURCES.json") + require(files, "SOURCES.json.files is empty; declare the source inventory") + base = require_key(sources, "base_commit", "SOURCES.json", "a 40-64 hex string pinning an exact base commit") + require(isinstance(base, str) and re.fullmatch(r"[0-9a-f]{40,64}", base), + "SOURCES.json.base_commit must be a 40-64 hex string pinning an exact base commit; found " + brief(base)) + task_base = require_key(task, "base_commit", "TASK.json", + "a 40-64 hex string equal to SOURCES.json.base_commit") + require(task_base == base, "TASK.json.base_commit " + brief(task_base) + + " must equal SOURCES.json.base_commit " + brief(base)) + for row in files: + require(isinstance(row, dict) and isinstance(row.get("path"), str), + "SOURCES.json.files rows must be JSON objects with a string path; offending row: " + brief(row)) + require(row.get("usage") in ("input", "provenance"), + "SOURCES.json.files row " + row["path"] + " must declare usage \"input\" or \"provenance\"; found " + + brief(row.get("usage"))) + undeclared = sorted({r["path"] for r in sources["files"] if r["usage"] == "input"} - set(inputs)) + require(not undeclared, + "every SOURCES.json files row with usage \"input\" must also be declared in VERIFY.json inputs" + + " (they drive CI invalidation); undeclared: " + ", ".join(undeclared)) + if check_pins: + catalog_name = "docs/research/agent-task-catalog.json" + require(catalog_name in {r["path"] for r in sources["files"]}, + "SOURCES.json.files must pin the originating task catalogue " + catalog_name + + " (a provenance row pinned at base_commit)") + names = [r["path"] for r in sources["files"]] + for name in names: + p = PurePosixPath(name) + require(not p.is_absolute() and ".." not in p.parts, "invalid source path") + specs = [source_pin(row, sources["base_commit"]) for row in sources["files"]] + require(all(spec == sources["base_commit"] + ":" + catalog_name + for name, spec in zip(names, specs) if name == catalog_name), + "originating task catalogue must be pinned at base_commit") + with closing(git_blobs(root, specs)) as objects: + for index, (obj, pinned) in enumerate(objects): + row = sources["files"][index] + name = row["path"] + require(obj == row["git_blob"], "source blob provenance mismatch: " + name) + require(hashlib.sha256(pinned).hexdigest() == row["sha256"], "source SHA256 provenance mismatch: " + name) + if row["usage"] == "input": + require(pinned == relative_file(root, name).read_bytes(), "working source drift: " + name) + if name == catalog_name: + catalog = json.loads(pinned) + cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task["task_id"]] + require(len(cards) == 1, + "TASK.json task_id " + brief(task["task_id"]) + " matches " + str(len(cards)) + + " cards in pinned " + catalog_name + " at base_commit " + sources["base_commit"] + + "; need exactly one") + original_card = require_key(task, "original_card", "TASK.json", + "the pinned catalogue card object (copy --print-card output verbatim)") + if original_card != cards[0]: + raise ValueError("original card in TASK.json must equal the pinned catalogue card object exactly; " + "copy it verbatim from " + catalog_name + " at base_commit " + sources["base_commit"] + + ": " + card_differences(original_card, cards[0])) + return task, manifest + + +def catalogue_card(root, package, task_id): + """The exact original_card object a hand-authored TASK.json must copy. + + Reads the catalogue blob pinned at the package base commit, never the + working tree, because validate() compares original_card against that pin. + """ + sources = json.loads(relative_file(root, package + "/SOURCES.json").read_text(encoding="utf-8")) + name = "docs/research/agent-task-catalog.json" + rows = require_key(sources, "files", "SOURCES.json") + row = next((r for r in rows if isinstance(r, dict) and r.get("path") == name), None) + require(row is not None, "SOURCES.json.files must pin the originating task catalogue " + name) + base = require_key(sources, "base_commit", "SOURCES.json", "a 40-64 hex string pinning an exact base commit") + spec = source_pin(row, base) + require(spec == base + ":" + name, "originating task catalogue must be pinned at base_commit") + catalog = json.loads(git(root, "show", spec)) + cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task_id] + require(len(cards) == 1, + "task_id " + brief(task_id) + " matches " + str(len(cards)) + + " cards in pinned " + name + " at base_commit " + base + "; need exactly one") + return cards[0] + + +_WINDOWS_GATE = r""" +import ctypes +from ctypes import wintypes +import json +import subprocess +import sys +api = ctypes.WinDLL('kernel32', use_last_error=True) +api.OpenEventW.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.LPCWSTR] +api.OpenEventW.restype = wintypes.HANDLE +api.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] +api.WaitForSingleObject.restype = wintypes.DWORD +api.CloseHandle.argtypes = [wintypes.HANDLE] +api.CloseHandle.restype = wintypes.BOOL +event = api.OpenEventW(0x00100000, False, sys.argv[1]) +if not event: + raise ctypes.WinError(ctypes.get_last_error()) +try: + if api.WaitForSingleObject(event, 30000) != 0: + raise RuntimeError('replay ownership gate was not released') +finally: + api.CloseHandle(event) +# No verification command or descendant starts until the parent owns this job. +sys.exit(subprocess.call(json.loads(sys.argv[2]))) +""" + +def run_windows_command(argv, cwd, timeout, env): + """Own ordinary verification descendants even after their leader exits. + + The gated launcher joins a kill-on-close Job Object before it may spawn + the requested command. A late taskkill cannot recover an exited leader's + tree. Job handles are not inherited and breakaway is not enabled. + """ + import ctypes + from ctypes import wintypes + import time + import uuid + + class BasicLimits(ctypes.Structure): + _fields_ = [("ProcessTime", ctypes.c_int64), ("JobTime", ctypes.c_int64), + ("Flags", wintypes.DWORD), ("MinWorkingSet", ctypes.c_size_t), + ("MaxWorkingSet", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), ("Priority", wintypes.DWORD), + ("Scheduling", wintypes.DWORD)] + + class IOCounts(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint64) for name in + ("ReadOps", "WriteOps", "OtherOps", "ReadBytes", "WriteBytes", "OtherBytes")] + + class ExtendedLimits(ctypes.Structure): + _fields_ = [("Basic", BasicLimits), ("IO", IOCounts), + ("ProcessMemory", ctypes.c_size_t), ("JobMemory", ctypes.c_size_t), + ("PeakProcessMemory", ctypes.c_size_t), ("PeakJobMemory", ctypes.c_size_t)] + + class Accounting(ctypes.Structure): + _fields_ = [(name, ctypes.c_int64) for name in + ("UserTime", "KernelTime", "PeriodUserTime", "PeriodKernelTime")] + [ + (name, wintypes.DWORD) for name in + ("PageFaults", "TotalProcesses", "ActiveProcesses", "TerminatedProcesses")] + + api = ctypes.WinDLL("kernel32", use_last_error=True) + + def native(name, result, *arguments): + fn = getattr(api, name) + fn.restype, fn.argtypes = result, arguments + return fn + + handle, dword, boolean = wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL + create_job = native("CreateJobObjectW", handle, ctypes.c_void_p, wintypes.LPCWSTR) + set_info = native("SetInformationJobObject", boolean, handle, ctypes.c_int, ctypes.c_void_p, dword) + assign = native("AssignProcessToJobObject", boolean, handle, handle) + terminate = native("TerminateJobObject", boolean, handle, wintypes.UINT) + query = native("QueryInformationJobObject", boolean, handle, ctypes.c_int, + ctypes.c_void_p, dword, ctypes.c_void_p) + create_event = native("CreateEventW", handle, ctypes.c_void_p, boolean, boolean, wintypes.LPCWSTR) + set_event = native("SetEvent", boolean, handle) + close = native("CloseHandle", boolean, handle) + + def checked(ok): + if not ok: + raise ctypes.WinError(ctypes.get_last_error()) + return ok + + job = checked(create_job(None, None)) + event, process, assigned = None, None, False + try: + limits = ExtendedLimits() + limits.Basic.Flags = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + checked(set_info(job, 9, ctypes.byref(limits), ctypes.sizeof(limits))) + name = "Local\\research-replay-" + uuid.uuid4().hex + event = checked(create_event(None, True, False, name)) + # A name collision must not attach to an already-signaled event. + if ctypes.get_last_error() == 183: + raise ValueError("replay ownership gate name already exists") + process = subprocess.Popen( + # The gate itself must not run sitecustomize/PYTHONPATH startup + # code before ownership. The requested command retains env/argv. + # Windows accepts Unicode argv directly. ASCII JSON escaping can + # expand an otherwise valid command past its UTF-16 launch limit. + [sys.executable, "-I", "-S", "-c", _WINDOWS_GATE, name, + json.dumps(argv, ensure_ascii=False)], + cwd=cwd, env=env, start_new_session=True) + checked(assign(job, int(process._handle))) + assigned = True + checked(set_event(event)) + try: + return process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + raise ValueError("verification timed out") from None + finally: + try: + if process is not None: + if assigned: + checked(terminate(job, 1)) + # Termination is asynchronous. Require the job to report + # no active processes before releasing its ownership handle. + deadline = time.monotonic() + 10 + while True: + state = Accounting() + checked(query(job, 1, ctypes.byref(state), ctypes.sizeof(state), None)) + if not state.ActiveProcesses: + break + if time.monotonic() >= deadline: + raise ValueError("verification process cleanup timed out") + time.sleep(0.01) + elif process.poll() is None: + # Assignment failed before the gate opened: no command child + # exists, so only this known gated launcher needs stopping. + process.kill() + process.wait(timeout=10) + finally: + try: + if event: + checked(close(event)) + finally: + checked(close(job)) + +def run_command(argv, cwd, timeout): + # Assertion checks must not disappear because of the caller's environment. + # Keep explicit manifest options (including deliberate -O controls) intact. + # PYTHONPATH precedes the standard library on a child's sys.path, so a stray + # same-named module could end a verifier with exit code 0 before any check. + env = os.environ.copy() + env.pop("PYTHONOPTIMIZE", None) + # Ambient inspection can turn a verifier exception into a successful REPL + # exit at terminal EOF. Keep explicit -i and the caller's stdin unchanged. + env.pop("PYTHONINSPECT", None) + env.pop("PYTHONPATH", None) + if os.name == "nt": + code = run_windows_command(argv, cwd, timeout, env) + require(code == 0, f"verification command exited {code}") + return + previous_handler = None + terminating = False + waiting = False + process = None + + def terminate(signum, frame): + nonlocal terminating + if not terminating: + terminating = True + # Only interrupt the protected wait. Launch/assignment and cleanup + # defer delivery; repeated signals cannot interrupt group cleanup. + if waiting: + raise SystemExit(128 + signum) + + # Own a group for POSIX cleanup of this command and its descendants. + try: + if (threading.current_thread() is threading.main_thread() + and signal.getsignal(signal.SIGTERM) == signal.SIG_DFL): + previous_handler = signal.SIG_DFL + signal.signal(signal.SIGTERM, terminate) + if terminating: + raise SystemExit(143) + process = subprocess.Popen(argv, cwd=cwd, env=env, start_new_session=True) + try: + try: + waiting = True + if terminating: + raise SystemExit(143) + code = process.wait(timeout=timeout) + finally: + waiting = False + except subprocess.TimeoutExpired: + raise ValueError("verification timed out") from None + require(code == 0, f"verification command exited {code}") + finally: + try: + if process is not None: + if hasattr(os, "killpg"): + # The leader may have exited while descendants hold its group. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + elif process.poll() is None: + subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], capture_output=True) + process.wait() + finally: + if previous_handler is not None: + signal.signal(signal.SIGTERM, previous_handler) + if terminating: + raise SystemExit(143) + + +def isolated_python_argv(argv, cache_directory): + """Place our cache option after interpreter flags, before the entry point. + + -B prevents writes, not reads of existing .pyc files. A fresh namespace + keeps preexisting source-adjacent and externally configured caches out of + this registered interpreter. It is not inherited by child interpreters. + """ + require(argv and argv[0] == sys.executable, "cache isolation requires the registered Python interpreter") + index = 1 + options = [argv[0]] + while index < len(argv): + option = argv[index] + if option in ("-", "--") or not option.startswith("-"): + break + if option == "--check-hash-based-pycs": + require(index + 1 < len(argv), "missing Python hash-cache option argument") + options.extend(argv[index:index + 2]) + index += 2 + continue + if option.startswith("--"): + options.append(option) + index += 1 + continue + # Python permits combined short flags, attached -c/-m payloads, and + # attached or separate -W/-X arguments. Stop scanning an option once + # its argument begins: letters inside that argument are not flags. + entry = False + consumed = 1 + replace_cache = False + for offset, flag in enumerate(option[1:], start=1): + if flag in "cm": + entry = True + break + if flag in "WX": + if offset == len(option) - 1: + require(index + 1 < len(argv), "missing Python interpreter option argument") + consumed = 2 + value = argv[index + 1] if consumed == 2 else option[offset + 1:] + if flag == "X" and value.split("=", 1)[0] == "pycache_prefix": + replace_cache = True + if offset > 1: + options.append(option[:offset]) + break + if entry: + break + if not replace_cache: + options.extend(argv[index:index + consumed]) + index += consumed + # CPython's effective prefix can use the first duplicate -X value even + # when sys._xoptions shows the last. Remove only earlier cache-prefix + # settings; script arguments remain untouched, even when spelled -X. + return [*options, "-X", "pycache_prefix=" + str(cache_directory), *argv[index:]] + + +def run_python_replay(argv, cwd, timeout): + with tempfile.TemporaryDirectory(prefix="research-replay-pycache-") as cache: + run_command(isolated_python_argv(argv, cache), cwd, timeout) + + +def run_package(root, package, install=False): + task, manifest = validate(root, package) + require(sys.version_info[:2] == (3, 12), "run replay with Python 3.12 (metadata check is version-independent)") + if install and manifest["requirements"]: + run_command([sys.executable, "-m", "pip", "install", "--disable-pip-version-check", *manifest["requirements"]], root, 300) + for requirement in manifest["requirements"]: + name, version = requirement.split("==") + require(importlib.metadata.version(name) == version, "dependency version mismatch: " + requirement) + # Coverage labels are UTF-8 metadata but a redirected Windows stdout may + # use cp1252. Escape unrepresentable display characters instead of aborting + # before replay. This changes only our stream, not child encoding or env. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(errors="backslashreplace") + for command in manifest["commands"]: + argv = [sys.executable if item == "{python}" else item.replace("{package}", package) for item in command["argv"]] + print("CHECK " + command["checks"], flush=True) + run_python_replay(argv, root, command["timeout_seconds"]) + print("PASS_REGISTERED_RESEARCH_PACKAGE " + package + " outcome=" + task["outcome"]) + + +def planning_manifests(root, tracked): + """Include deliberate sparse omissions without disguising deletions or drift. + + The fallback follows the chooser's PR3102 contract: one stage-zero regular + skip-worktree index entry must match the exact HEAD mode and blob. This is + declaration discovery only, not permission to replay absent package code. + """ + committed = {name for name in tracked + if name.startswith("research/") and name.count("/") == 2 + and name.endswith("/VERIFY.json")} + # Explicit scandir retains error detection; only an absent directory is OK. + try: + with os.scandir(root / "research") as entries: + local = {Path(entry.path) / "VERIFY.json" for entry in entries} + except FileNotFoundError: + local = set() + present, missing = {}, set() + for path in sorted(local | {root / name for name in committed}): + name = path.relative_to(root).as_posix() + symlink_free_relative_path(root, name) + try: + mode = path.lstat().st_mode + except (FileNotFoundError, NotADirectoryError): + if name in committed: + missing.add(name) + continue + require(stat.S_ISREG(mode), "non-regular replay manifest: " + name) + present[name] = path + fallback = {} + if missing: + indexed, objects = {}, {} + for record in git(root, "ls-files", "-t", "--stage", "-z", "--", "research").split(b"\0"): + if not record: + continue + header, raw_name = record.split(b"\t", 1) + name = raw_name.decode("utf-8") + if name in missing: + indexed.setdefault(name, []).append(header.split()) + for record in git(root, "ls-tree", "-r", "-z", "HEAD", "--", "research").split(b"\0"): + if not record: + continue + header, raw_name = record.split(b"\t", 1) + name = raw_name.decode("utf-8") + if name in missing: + objects[name] = header.split() + for name in sorted(missing): + metadata = objects.get(name, []) + require(len(metadata) == 3 and metadata[0] in (b"100644", b"100755") + and metadata[1] == b"blob" + and indexed.get(name) == [[b"S", metadata[0], metadata[2], b"0"]], + "missing replay manifest is not an unchanged sparse HEAD input: " + name + + "; materialize it or commit its intended deletion") + # A regular file occupying the package directory is not an omission. + require(not (root / name).parent.exists() or (root / name).parent.is_dir(), + "non-directory replay package: " + name) + names = sorted(missing) + with closing(git_blobs(root, [objects[n][2].decode("ascii") for n in names])) as rows: + for (oid, raw), name in zip(rows, names): + require(oid.encode("ascii") == objects[name][2], "sparse manifest blob mismatch: " + name) + fallback[name] = raw + # Preserve Path's component ordering: package "a" precedes "a-extra". + for name in sorted(present.keys() | fallback.keys(), key=lambda name: PurePosixPath(name).parts): + yield root / name, fallback[name] if name in fallback else present[name].read_bytes(), name in fallback + + +def plan_details(root, changed): + """One selection pass with exact causes; no replay or acceptance decision.""" + changed = set(changed) + # A removed input is absent from HEAD, including an old case-only rename + # spelling on case-insensitive filesystems. Git's exact changed paths remain + # eligible so its consumer is selected, rather than rejected or skipped. + head_tracked = tracked_input_paths(root) + tracked = head_tracked | changed + replay_changed = sorted(changed.intersection({ + "scripts/research_package.py", ".github/workflows/research-packages.yml", + ".github/workflows/research-package-batch.yml"})) + # Group literal Git paths once. Keep removed paths and exact spellings. + changed_packages = {} + for name in sorted(changed): + parts = name.split("/", 2) + if len(parts) == 3 and parts[0] == "research": + changed_packages.setdefault("/".join(parts[:2]), []).append(name) + packages = [] + sparse_count = 0 + for path, raw, sparse in planning_manifests(root, head_tracked): + sparse_count += sparse + package = path.parent.relative_to(root).as_posix() + manifest = json.loads(raw.decode("utf-8")) + # Parse only for selection; validate fully in the isolated replay job. + inputs = manifest.get("inputs", []) + require(isinstance(inputs, list), "declare input paths for CI invalidation") + for name in inputs: + # Do not require existence: a deleted input must still trigger replay. + symlink_free_relative_path(root, name) + require_tracked_input(name, tracked) + causes = { + "declared_inputs": sorted(changed.intersection(inputs)), + "package_files": changed_packages.get(package, []), + "shared_controls": replay_changed, + } + if any(causes.values()): + packages.append({"package": package, "causes": causes, + "manifest_sha256": hashlib.sha256(raw).hexdigest()}) + if sparse_count: + print(f"Planning includes {sparse_count} unchanged sparse manifests from HEAD; " + "materialize packages and inputs before replay.", file=sys.stderr) + return packages + + +def plan(root, changed): + return [row["package"] for row in plan_details(root, changed)] + + +def write_plan_explanation(path, base, head, changed, details): + """Publish a complete sidecar before stdout/GitHub matrix publication.""" + matched = {name for row in details for names in row["causes"].values() for name in names} + report = { + "schema_version": 1, "base": base, "head": head, + "mode": "all-tracked" if base is None else "commit-diff", + "scope": "Selection from working or authenticated sparse HEAD manifests, not replay or mathematical acceptance. " + "Manifest hashes bind the declarations read; checkout must remain stable.", + "changed_paths": sorted(set(changed) - {""}), "selected": details, + "unmatched_paths": sorted(set(changed) - matched - {""}), + } + fd, temporary = tempfile.mkstemp(prefix=".replay-plan-", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(report, stream, indent=2) + stream.write("\n") + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + + +def replay_matrix(packages, batch_size=None): + """Retain the flat CLI format, or group complete coverage into bounded matrices.""" + if batch_size is None: + return {"include": [{"package": p} for p in packages]} + require(type(batch_size) is int and 1 <= batch_size <= 256, + "matrix batch size must be 1..256") + batches = [packages[i:i + batch_size] for i in range(0, len(packages), batch_size)] + require(len(batches) <= 256, + "more than 256 matrix batches required; no packages were truncated") + return {"include": [{"packages": batch} for batch in batches]} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--package") + ap.add_argument("--check-only", action="store_true") + ap.add_argument("--print-card", metavar="TASK_ID", + help="print the exact pinned catalogue card object to copy into TASK.json original_card") + ap.add_argument("--install-deps", action="store_true", help="explicit opt-in to install manifest's allowed pinned dependencies") + ap.add_argument("--changed-since", help="plan affected opt-in packages relative to this Git commit") + ap.add_argument("--github-output", type=Path) + ap.add_argument("--explain-plan", type=Path, + help="write exact replay-selection causes to JSON; matrix output stays unchanged") + ap.add_argument("--matrix-batch-size", type=int, + help="group planned packages for bounded reusable-workflow matrices (1..256)") + args = ap.parse_args() + if args.explain_plan and (not args.changed_since or args.package or args.print_card): + ap.error("--explain-plan requires --changed-since without --package or --print-card") + if args.explain_plan and args.github_output and args.explain_plan.resolve() == args.github_output.resolve(): + ap.error("--explain-plan and --github-output must use different files") + try: + if args.print_card: + require(args.package, + "--print-card needs --package research/ to resolve its pinned catalogue") + sys.stdout.write(json.dumps(catalogue_card(ROOT, args.package, args.print_card), indent=2) + "\n") + elif args.changed_since: + if set(args.changed_since) == {"0"}: + base = None + changed = git(ROOT, "ls-files", "-z").decode().split("\0") + else: + base = git(ROOT, "rev-parse", "--verify", args.changed_since + "^{commit}").decode().strip() + # A rename must invalidate both the removed input and its new path. + changed = git(ROOT, "diff", "--no-renames", "--name-only", "-z", base, "HEAD").decode().split("\0") + details = plan_details(ROOT, changed) + packages = [row["package"] for row in details] + matrix = json.dumps(replay_matrix(packages, args.matrix_batch_size)) + if args.explain_plan: + head = git(ROOT, "rev-parse", "--verify", "HEAD^{commit}").decode().strip() + write_plan_explanation(args.explain_plan, base, head, changed, details) + print(matrix) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as out: + out.write(f"matrix={matrix}\ncount={len(packages)}\n") + elif args.package: + if args.check_only: + validate(ROOT, args.package) + print("PASS_PACKAGE_METADATA_AND_PINS (not a mathematical acceptance check)") + else: + run_package(ROOT, args.package, args.install_deps) + else: + ap.error("use --package, --print-card or --changed-since") + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError, importlib.metadata.PackageNotFoundError) as exc: + ap.exit(1, str(exc) + "\n") + + +if __name__ == "__main__": + main() diff --git a/research/windows-git-stream-ownership/evidence/linux-portable.json b/research/windows-git-stream-ownership/evidence/linux-portable.json new file mode 100644 index 0000000000..db3b15fe59 --- /dev/null +++ b/research/windows-git-stream-ownership/evidence/linux-portable.json @@ -0,0 +1,85 @@ +{ + "assignment_failure_control_executed": false, + "cases": [ + { + "has_child": false, + "mode": "header", + "observation": {}, + "owned_command": false, + "result": { + "calls": 1, + "error": "TimeoutExpired" + }, + "stream_adapter": false + }, + { + "has_child": false, + "mode": "payload", + "observation": {}, + "owned_command": false, + "result": { + "calls": 1, + "error": "TimeoutExpired" + }, + "stream_adapter": false + }, + { + "has_child": false, + "mode": "terminator", + "observation": {}, + "owned_command": false, + "result": { + "calls": 1, + "error": "TimeoutExpired" + }, + "stream_adapter": false + }, + { + "has_child": false, + "mode": "success", + "observation": {}, + "owned_command": false, + "result": { + "calls": 1, + "exact": true + }, + "stream_adapter": false + }, + { + "has_child": false, + "mode": "malformed", + "observation": {}, + "owned_command": false, + "result": { + "calls": 1, + "error": "ValueError" + }, + "stream_adapter": false + }, + { + "has_child": false, + "mode": "suspended", + "observation": {}, + "owned_command": false, + "result": { + "calls": 1, + "exhausted": true, + "first_exact": true, + "second_exact": true + }, + "stream_adapter": false + } + ], + "implementation_hashes": { + "owned_pipe.py": "01f597c8ecb955b2ef7faa8f0650e67c0eca5c0d5f6a817a181c536bef9c37aa", + "verify.py": "c7d6f6ce6fe3931f2daf6def507c379a3644b6058201e895a959b5f7034098a0" + }, + "optimized_driver_and_workers": false, + "platform": "Linux-6.6.114.1-microsoft-standard-WSL2-x86_64-with-glibc2.39", + "python": "3.12.3 (main, Jun 19 2026, 12:46:00) [GCC 13.3.0]", + "real_git_binary_duplicate_control": true, + "response_timeout_seconds": 0.8, + "source_sha256": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6", + "status": "PASS_PORTABLE_CONTROLS_WINDOWS_NOT_RUN", + "windows_observations_executed": false +} diff --git a/research/windows-git-stream-ownership/evidence/windows-normal.json b/research/windows-git-stream-ownership/evidence/windows-normal.json new file mode 100644 index 0000000000..f6b716639d --- /dev/null +++ b/research/windows-git-stream-ownership/evidence/windows-normal.json @@ -0,0 +1,348 @@ +{ + "status": "PASS_WINDOWS_GIT_STREAM_OWNERSHIP", + "source_sha256": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6", + "implementation_hashes": { + "verify.py": "c7d6f6ce6fe3931f2daf6def507c379a3644b6058201e895a959b5f7034098a0", + "owned_pipe.py": "01f597c8ecb955b2ef7faa8f0650e67c0eca5c0d5f6a817a181c536bef9c37aa" + }, + "platform": "Windows-11-10.0.26300-SP0", + "python": "3.12.0 (tags/v3.12.0:0fb18b0, Oct 2 2023, 13:03:39) [MSC v.1935 64 bit (AMD64)]", + "optimized_driver_and_workers": false, + "response_timeout_seconds": 0.8, + "windows_observations_executed": true, + "assignment_failure_control_executed": true, + "real_git_binary_duplicate_control": true, + "cases": [ + { + "mode": "header", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "payload", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "terminator", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "success", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "exact": true + } + }, + { + "mode": "malformed", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "ValueError" + } + }, + { + "mode": "suspended", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "exhausted": true, + "first_exact": true, + "second_exact": true + } + }, + { + "mode": "header", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "payload", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "terminator", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "write", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "success", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 1, + "exact": true + } + }, + { + "mode": "malformed", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "ValueError" + } + }, + { + "mode": "early", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 1, + "first_exact": true + } + }, + { + "mode": "fallback", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 2, + "exact": true + } + }, + { + "mode": "header", + "has_child": true, + "owned_command": true, + "stream_adapter": false, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "error": "verification timed out" + } + }, + { + "mode": "success", + "has_child": true, + "owned_command": true, + "stream_adapter": false, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "code": 0 + } + }, + { + "mode": "header", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "payload", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "terminator", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "write", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "success", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "exact": true + } + }, + { + "mode": "malformed", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "ValueError" + } + }, + { + "mode": "early", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "first_exact": true + } + }, + { + "mode": "fallback", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 2, + "exact": true + } + }, + { + "mode": "suspended", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "exhausted": true, + "first_exact": true, + "second_exact": true + } + } + ] +} diff --git a/research/windows-git-stream-ownership/evidence/windows-optimized.json b/research/windows-git-stream-ownership/evidence/windows-optimized.json new file mode 100644 index 0000000000..27d868c9c0 --- /dev/null +++ b/research/windows-git-stream-ownership/evidence/windows-optimized.json @@ -0,0 +1,348 @@ +{ + "status": "PASS_WINDOWS_GIT_STREAM_OWNERSHIP", + "source_sha256": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6", + "implementation_hashes": { + "verify.py": "c7d6f6ce6fe3931f2daf6def507c379a3644b6058201e895a959b5f7034098a0", + "owned_pipe.py": "01f597c8ecb955b2ef7faa8f0650e67c0eca5c0d5f6a817a181c536bef9c37aa" + }, + "platform": "Windows-11-10.0.26300-SP0", + "python": "3.12.0 (tags/v3.12.0:0fb18b0, Oct 2 2023, 13:03:39) [MSC v.1935 64 bit (AMD64)]", + "optimized_driver_and_workers": true, + "response_timeout_seconds": 0.8, + "windows_observations_executed": true, + "assignment_failure_control_executed": true, + "real_git_binary_duplicate_control": true, + "cases": [ + { + "mode": "header", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "payload", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "terminator", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "success", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "exact": true + } + }, + { + "mode": "malformed", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "error": "ValueError" + } + }, + { + "mode": "suspended", + "has_child": false, + "owned_command": false, + "stream_adapter": false, + "observation": {}, + "result": { + "calls": 1, + "exhausted": true, + "first_exact": true, + "second_exact": true + } + }, + { + "mode": "header", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "payload", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "terminator", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "write", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true, + "reader_pending_after_watchdog": true, + "no_result_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "success", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 1, + "exact": true + } + }, + { + "mode": "malformed", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "ValueError" + } + }, + { + "mode": "early", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 1, + "first_exact": true + } + }, + { + "mode": "fallback", + "has_child": true, + "owned_command": false, + "stream_adapter": false, + "observation": { + "leader_exited": true, + "child_alive_before_cleanup": true + }, + "result": { + "calls": 2, + "exact": true + } + }, + { + "mode": "header", + "has_child": true, + "owned_command": true, + "stream_adapter": false, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "error": "verification timed out" + } + }, + { + "mode": "success", + "has_child": true, + "owned_command": true, + "stream_adapter": false, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "code": 0 + } + }, + { + "mode": "header", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "payload", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "terminator", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "write", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "TimeoutExpired" + } + }, + { + "mode": "success", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "exact": true + } + }, + { + "mode": "malformed", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "error": "ValueError" + } + }, + { + "mode": "early", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "first_exact": true + } + }, + { + "mode": "fallback", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 2, + "exact": true + } + }, + { + "mode": "suspended", + "has_child": true, + "owned_command": false, + "stream_adapter": true, + "observation": { + "all_handles_stopped_before_cleanup": true + }, + "result": { + "calls": 1, + "exhausted": true, + "first_exact": true, + "second_exact": true + } + } + ] +} diff --git a/research/windows-git-stream-ownership/owned_pipe.py b/research/windows-git-stream-ownership/owned_pipe.py new file mode 100644 index 0000000000..8150b9a79d --- /dev/null +++ b/research/windows-git-stream-ownership/owned_pipe.py @@ -0,0 +1,153 @@ +"""Opt-in Windows stream lifetime for the source-pinned Git-reader experiment. + +The native structures, gate and ownership protocol are adapted from ywz65's +run_windows_command in the frozen research_package.py. Its notice and exact +source are retained beside this module. Streaming interface by Mihir Srivastava +with substantial OpenAI Codex assistance; Apache-2.0. +""" +from contextlib import contextmanager +import ctypes +from ctypes import wintypes +import json +import subprocess +import sys +import threading +import time +import uuid + +REAL_POPEN = subprocess.Popen + + +@contextmanager +def windows_pipe(argv, gate, **kwargs): + """Yield a gated process and an idempotent, thread-safe whole-job stop. + + Only the job handle owner may release the gate. The requested command's + stdin/stdout are inherited through the gate without Python-level copying. + """ + class BasicLimits(ctypes.Structure): + _fields_ = [("ProcessTime", ctypes.c_int64), ("JobTime", ctypes.c_int64), + ("Flags", wintypes.DWORD), ("MinWorkingSet", ctypes.c_size_t), + ("MaxWorkingSet", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), ("Priority", wintypes.DWORD), + ("Scheduling", wintypes.DWORD)] + + class IOCounts(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint64) for name in + ("ReadOps", "WriteOps", "OtherOps", "ReadBytes", "WriteBytes", "OtherBytes")] + + class ExtendedLimits(ctypes.Structure): + _fields_ = [("Basic", BasicLimits), ("IO", IOCounts), + ("ProcessMemory", ctypes.c_size_t), ("JobMemory", ctypes.c_size_t), + ("PeakProcessMemory", ctypes.c_size_t), ("PeakJobMemory", ctypes.c_size_t)] + + class Accounting(ctypes.Structure): + _fields_ = [(name, ctypes.c_int64) for name in + ("UserTime", "KernelTime", "PeriodUserTime", "PeriodKernelTime")] + [ + (name, wintypes.DWORD) for name in + ("PageFaults", "TotalProcesses", "ActiveProcesses", "TerminatedProcesses")] + + api = ctypes.WinDLL("kernel32", use_last_error=True) + + def native(name, result, *arguments): + fn = getattr(api, name) + fn.restype, fn.argtypes = result, arguments + return fn + + handle, dword, boolean = wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL + create_job = native("CreateJobObjectW", handle, ctypes.c_void_p, wintypes.LPCWSTR) + set_info = native("SetInformationJobObject", boolean, handle, ctypes.c_int, ctypes.c_void_p, dword) + assign = native("AssignProcessToJobObject", boolean, handle, handle) + terminate = native("TerminateJobObject", boolean, handle, wintypes.UINT) + query = native("QueryInformationJobObject", boolean, handle, ctypes.c_int, + ctypes.c_void_p, dword, ctypes.c_void_p) + create_event = native("CreateEventW", handle, ctypes.c_void_p, boolean, boolean, wintypes.LPCWSTR) + set_event = native("SetEvent", boolean, handle) + close = native("CloseHandle", boolean, handle) + + def checked(ok): + if not ok: + raise ctypes.WinError(ctypes.get_last_error()) + return ok + + job = checked(create_job(None, None)) + event, process, assigned = None, None, False + stopped = False + lock = threading.Lock() + + def stop(): + nonlocal stopped + with lock: + if stopped: + return + checked(terminate(job, 1)) + deadline = time.monotonic() + 10 + while True: + state = Accounting() + checked(query(job, 1, ctypes.byref(state), ctypes.sizeof(state), None)) + if not state.ActiveProcesses: + break + if time.monotonic() >= deadline: + raise RuntimeError("owned Git process cleanup timed out") + time.sleep(0.01) + stopped = True + + try: + limits = ExtendedLimits() + limits.Basic.Flags = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + checked(set_info(job, 9, ctypes.byref(limits), ctypes.sizeof(limits))) + name = "Local\\research-git-stream-" + uuid.uuid4().hex + event = checked(create_event(None, True, False, name)) + if ctypes.get_last_error() == 183: + raise ValueError("Git ownership gate name already exists") + process = REAL_POPEN([sys.executable, "-I", "-S", "-c", gate, name, + json.dumps(argv, ensure_ascii=False)], **kwargs) + checked(assign(job, int(process._handle))) + assigned = True + checked(set_event(event)) + yield process, stop + finally: + try: + if process is not None: + if assigned: + stop() + elif process.poll() is None: + process.kill() + process.wait(timeout=10) + finally: + try: + if process is not None: + # In the actual reader these are already closed. Setup + # failure can reach here before the reader owns any streams. + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None: + stream.close() + finally: + try: + if event: + checked(close(event)) + finally: + checked(close(job)) + + +class OwnedPopen: + """The Popen subset used by the frozen reader, with job-aware kill/exit. + + This is an experimental reader adapter, not a general subprocess replacement. + The caller must use the context manager, as the actual reader does. + """ + def __init__(self, argv, gate, **kwargs): + self.context = windows_pipe(argv, gate, **kwargs) + self.process, self.stop = self.context.__enter__() + + def __getattr__(self, name): + return getattr(self.process, name) + + def kill(self): + self.stop() + + def __enter__(self): + return self + + def __exit__(self, *exception): + return self.context.__exit__(*exception) diff --git a/research/windows-git-stream-ownership/verify.py b/research/windows-git-stream-ownership/verify.py new file mode 100644 index 0000000000..2451684d22 --- /dev/null +++ b/research/windows-git-stream-ownership/verify.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +"""Check an opt-in streaming owner against the frozen Windows Git reader. + +Mihir Srivastava, with substantial OpenAI Codex assistance. Apache-2.0. +Readiness/lifetime design follows JoyboyBrian's POSIX reader controls and +ywz65's native process-handle and Job Object controls; see README.md. +""" +import argparse +import ctypes +from ctypes import wintypes +import hashlib +import json +import os +from pathlib import Path +import platform +import signal +import subprocess +import sys +import tempfile +import time +import types +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +TIMEOUT = 0.8 +PAYLOAD = b"\0\xff\r\n" +OID = "a" * 40 + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def python(): + return [sys.executable, "-I", "-W", "error", "-B"] + (["-O"] if sys.flags.optimize else []) + + +def write_json(path, value): + pending = path.with_suffix(".pending") + pending.write_text(json.dumps(value, sort_keys=True) + "\n", encoding="utf-8") + pending.replace(path) + + +def wait_for(predicate, seconds=20): + until = time.monotonic() + seconds + while not predicate(): + if time.monotonic() >= until: + raise AssertionError("fixture readiness/completion timed out") + time.sleep(0.01) + + +def load_baseline(): + source = HERE / "baseline/research_package.py" + rows = json.loads((HERE / "SOURCES.json").read_bytes())["files"] + expected = next(r["sha256"] for r in rows if r["path"] == "scripts/research_package.py") + require(hashlib.sha256(source.read_bytes()).hexdigest() == expected, + "frozen source bytes differ from the recorded Git source") + module = types.ModuleType("frozen_reader") + module.__file__ = str(source) + # Compile the authenticated bytes, never a possibly stale adjacent .pyc. + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + module.subprocess = types.SimpleNamespace(**vars(subprocess)) + return module, expected + + +def load_adapter(): + source = HERE / "owned_pipe.py" + module = types.ModuleType("owned_pipe") + module.__file__ = str(source) + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + return module + + +class ProcessHandle: + """Retain the actual process object, including after its PID is reused.""" + def __init__(self, pid): + self.api = ctypes.WinDLL("kernel32", use_last_error=True) + for name, result, args in ( + ("OpenProcess", wintypes.HANDLE, [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]), + ("WaitForSingleObject", wintypes.DWORD, [wintypes.HANDLE, wintypes.DWORD]), + ("TerminateProcess", wintypes.BOOL, [wintypes.HANDLE, wintypes.UINT]), + ("CloseHandle", wintypes.BOOL, [wintypes.HANDLE]), + ): + fn = getattr(self.api, name) + fn.restype, fn.argtypes = result, args + self.handle = self.api.OpenProcess(0x00100001, False, pid) + if not self.handle: + raise ctypes.WinError(ctypes.get_last_error()) + + def alive(self): + result = self.api.WaitForSingleObject(self.handle, 0) + require(result in (0, 258), "native process observation failed") + return result == 258 + + def stop(self): + if self.alive() and not self.api.TerminateProcess(self.handle, 1): + require(not self.alive(), "native fixture termination failed") + require(self.api.WaitForSingleObject(self.handle, 5000) == 0, + "fixture process failed to terminate") + + def close(self): + require(self.api.CloseHandle(self.handle), "native handle close failed") + + +def child(directory): + write_json(directory / "child.json", {"pid": os.getpid()}) + # Self-exit is a final guard, far beyond the measured observation window. + wait_for((directory / "release-child").exists, 60) + + +def leader(directory, mode, has_child): + write_json(directory / "leader.json", {"pid": os.getpid()}) + descendant = None + if has_child: + # Explicit standard streams make handle inheritance part of the fixture. + descendant = subprocess.Popen([*python(), str(HERE / "verify.py"), "--child", str(directory)], + stdin=sys.stdin, stdout=sys.stdout, stderr=subprocess.DEVNULL) + wait_for((directory / "child.json").exists) + (directory / "ready").touch() + wait_for((directory / "release-leader").exists) + if mode == "write": + os._exit(0) + while sys.stdin.buffer.read(1) not in (b"\0", b"\n", b""): + pass + reply = OID.encode() + b" blob 4\n" + PAYLOAD + b"\n" + pieces = {"header": b"", "payload": reply[:50], "terminator": reply[:-1], + "malformed": b"bad\n"} + sys.stdout.buffer.write(pieces.get(mode, reply)) + sys.stdout.buffer.flush() + if mode in ("header", "payload", "terminator") and not has_child: + time.sleep(60) # The reader's direct-child watchdog must end this. + if mode == "early": + time.sleep(60) + if mode == "suspended": + while sys.stdin.buffer.read(1) not in (b"\0", b"\n", b""): + pass + sys.stdout.buffer.write(reply) + sys.stdout.buffer.flush() + # Intentional unjoined process exit is the behavior under investigation. + os._exit(0) + + +def worker(directory, mode, has_child, repaired=False): + baseline, _ = load_baseline() + baseline.GIT_BLOB_RESPONSE_TIMEOUT = TIMEOUT + adapter = load_adapter() + real_popen = ((lambda argv, **kwargs: adapter.OwnedPopen(argv, baseline._WINDOWS_GATE, **kwargs)) + if repaired else subprocess.Popen) + real_stop = baseline._stop_git_blob_process + calls = [] + + def start(argv, **kwargs): + calls.append(argv) + if mode == "fallback" and len(calls) == 1: + return real_popen([*python(), "-c", "raise SystemExit(129)"], **kwargs) + process = real_popen([*python(), str(HERE / "verify.py"), "--leader", + str(directory), mode, str(int(has_child))], **kwargs) + write_json(directory / "launcher.json", {"pid": process.pid}) + wait_for((directory / "observed").exists) + (directory / "release-leader").touch() + return process + + def stop(process): + real_stop(process) + (directory / "watchdog-fired").touch() + + result = {} + with patch.object(baseline.subprocess, "Popen", side_effect=start), \ + patch.object(baseline, "_stop_git_blob_process", side_effect=stop): + try: + specs = ["x" * 2_000_000] if mode == "write" else ["HEAD:first"] + if mode == "suspended": + specs.append("HEAD:second") + rows = baseline.git_blobs(directory, specs) + if mode in ("early", "suspended"): + result["first_exact"] = next(rows) == (OID, PAYLOAD) + if mode == "suspended": + time.sleep(1.2) # Longer than the per-response deadline. + result["second_exact"] = next(rows) == (OID, PAYLOAD) + result["exhausted"] = list(rows) == [] + rows.close() + else: + result["exact"] = list(rows) == [(OID, PAYLOAD)] + except Exception as error: + result["error"] = type(error).__name__ + result["calls"] = len(calls) + write_json(directory / "result.json", result) + + +def owned_worker(directory, mode): + baseline, _ = load_baseline() + argv = [*python(), str(HERE / "verify.py"), "--worker", str(directory), mode, "1"] + result = {} + try: + result["code"] = baseline.run_windows_command(argv, directory, 15, os.environ.copy()) + except ValueError as error: + result["error"] = str(error) + write_json(directory / "owned-result.json", result) + + +def assignment_failure_control(): + baseline, _ = load_baseline() + adapter = load_adapter() + real_dll = ctypes.WinDLL + launched = [] + + def denied(*args): + ctypes.set_last_error(5) + return 0 + + class DeniedAssignment: + def __init__(self, *args, **kwargs): + self.api = real_dll(*args, **kwargs) + + def __getattr__(self, name): + return denied if name == "AssignProcessToJobObject" else getattr(self.api, name) + + def launch(*args, **kwargs): + process = subprocess.Popen(*args, **kwargs) + launched.append(process) + return process + + with tempfile.TemporaryDirectory(prefix="denied stream owner ") as tmp: + marker = Path(tmp) / "command-ran" + argv = [*python(), "-c", "from pathlib import Path; import sys; Path(sys.argv[1]).touch()", str(marker)] + with patch.object(ctypes, "WinDLL", DeniedAssignment), patch.object(adapter, "REAL_POPEN", launch): + try: + with adapter.OwnedPopen(argv, baseline._WINDOWS_GATE, + stdin=subprocess.PIPE, stdout=subprocess.PIPE): + raise AssertionError("assignment failure yielded a live reader") + except OSError as error: + require(error.winerror == 5, "wrong assignment error") + require(len(launched) == 1 and launched[0].poll() is not None, "unassigned gate survived") + require(not marker.exists(), "requested command ran before ownership") + require(launched[0].stdin.closed and launched[0].stdout.closed, "setup leaked stream handles") + return True + + +def exercise(mode, has_child=True, owned=False, repaired=False): + handles = [] + with tempfile.TemporaryDirectory(prefix="git lifetime ") as tmp: + directory = Path(tmp) + args = (["--owned-worker", tmp, mode] if owned else + ["--worker", tmp, mode, str(int(has_child)), str(int(repaired))]) + with (directory / "worker.log").open("wb") as log: + process = subprocess.Popen([*python(), str(HERE / "verify.py"), *args], stdout=log, stderr=log) + try: + wait_for(lambda: (directory / "ready").exists() or process.poll() is not None) + require((directory / "ready").exists(), + "worker exited before readiness: " + (directory / "worker.log").read_text()) + if sys.platform == "win32": + for filename in (["leader.json", "child.json"] if has_child else ["leader.json"]): + pid = json.loads((directory / filename).read_bytes())["pid"] + handles.append(ProcessHandle(pid)) + if repaired: + handles.append(ProcessHandle(json.loads((directory / "launcher.json").read_bytes())["pid"])) + require(all(h.alive() for h in handles), "process exited before retained observation") + (directory / "observed").touch() + stall = mode in ("header", "payload", "terminator", "write") + if stall and has_child and not owned and not repaired: + wait_for((directory / "watchdog-fired").exists) + wait_for(lambda: not handles[0].alive()) + time.sleep(1.2) # Reader still blocked after its watchdog has fired. + evidence = {"leader_exited": not handles[0].alive(), + "child_alive_before_cleanup": handles[1].alive(), + "reader_pending_after_watchdog": process.poll() is None, + "no_result_before_cleanup": not (directory / "result.json").exists()} + require(all(evidence.values()), "expected descendant-held-pipe gap was not observed") + # Only now release the pipe and check the deferred exception. + handles[1].stop() + else: + process.wait(timeout=22) + evidence = {} + process.wait(timeout=5) + require(process.returncode == 0, "worker failed: " + (directory / "worker.log").read_text()) + if owned: + wait_for(lambda: not any(h.alive() for h in handles)) + result = json.loads((directory / "owned-result.json").read_bytes()) + expected = {"error": "verification timed out"} if stall else {"code": 0} + require(result == expected, "whole-command control returned the wrong status") + evidence["all_handles_stopped_before_cleanup"] = True + if stall: + require(not (directory / "result.json").exists(), "reader unexpectedly returned before owner timeout") + else: + result = json.loads((directory / "result.json").read_bytes()) + expected = ({"error": "TimeoutExpired", "calls": 1} if stall else + {"error": "ValueError", "calls": 1} if mode == "malformed" else + {"first_exact": True, "calls": 1} if mode == "early" else + {"first_exact": True, "second_exact": True, "exhausted": True, "calls": 1} + if mode == "suspended" else + {"exact": True, "calls": 2 if mode == "fallback" else 1}) + require(result == expected, "reader result differs: " + repr(result)) + if repaired: + wait_for(lambda: not any(h.alive() for h in handles)) + evidence["all_handles_stopped_before_cleanup"] = True + elif has_child and not stall: + require(not handles[0].alive(), "leader was not reaped") + require(handles[1].alive(), "expected post-return descendant leak was not observed") + evidence.update(leader_exited=True, child_alive_before_cleanup=True) + return {"mode": mode, "has_child": has_child, "owned_command": owned, "stream_adapter": repaired, + "observation": evidence, "result": result} + finally: + # Failure containment is never counted as successful reader cleanup. + (directory / "release-child").touch() + (directory / "observed").touch() + for handle in reversed(handles): + try: + handle.stop() + finally: + handle.close() + if os.name == "posix" and (directory / "leader.json").exists(): + pid = json.loads((directory / "leader.json").read_bytes())["pid"] + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + parser.add_argument("--require-windows", action="store_true", + help="fail rather than run only portable controls on another OS") + args = parser.parse_args() + require(not args.require_windows or sys.platform == "win32", "native Windows execution required") + baseline, source_hash = load_baseline() + rows = [exercise(mode, has_child=False) for mode in + ("header", "payload", "terminator", "success", "malformed", "suspended")] + with tempfile.TemporaryDirectory(prefix="real git blobs ") as tmp: + root = Path(tmp) + baseline.git(root, "init", "-q") + (root / "data").write_bytes(PAYLOAD) + oid = baseline.git(root, "hash-object", "-w", "--", "data").decode().strip() + require(list(baseline.git_blobs(root, [oid, oid])) == [(oid, PAYLOAD)] * 2, + "real Git exact binary/duplicate control failed") + if sys.platform == "win32": + rows += [exercise(mode) for mode in + ("header", "payload", "terminator", "write", "success", "malformed", "early", "fallback")] + rows += [exercise(mode, owned=True) for mode in ("header", "success")] + rows += [exercise(mode, repaired=True) for mode in + ("header", "payload", "terminator", "write", "success", "malformed", "early", "fallback", "suspended")] + assignment_failure_control() + adapter = load_adapter() + baseline.subprocess.Popen = lambda argv, **kw: adapter.OwnedPopen(argv, baseline._WINDOWS_GATE, **kw) + with tempfile.TemporaryDirectory(prefix="owned real Git ") as tmp: + root = Path(tmp) + baseline.git(root, "init", "-q") + (root / "data").write_bytes(PAYLOAD) + oid = baseline.git(root, "hash-object", "-w", "--", "data").decode().strip() + require(list(baseline.git_blobs(root, [oid, oid])) == [(oid, PAYLOAD)] * 2, + "owned real Git binary/duplicate control failed") + result = {"status": ("PASS_WINDOWS_GIT_STREAM_OWNERSHIP" if sys.platform == "win32" else + "PASS_PORTABLE_CONTROLS_WINDOWS_NOT_RUN"), "source_sha256": source_hash, + "implementation_hashes": {name: hashlib.sha256((HERE / name).read_bytes()).hexdigest() + for name in ("verify.py", "owned_pipe.py")}, + "platform": platform.platform(), "python": sys.version, + "optimized_driver_and_workers": bool(sys.flags.optimize), "response_timeout_seconds": TIMEOUT, + "windows_observations_executed": sys.platform == "win32", + "assignment_failure_control_executed": sys.platform == "win32", + "real_git_binary_duplicate_control": True, "cases": rows} + if args.output: + write_json(args.output, result) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "--child": + child(Path(sys.argv[2])) + elif len(sys.argv) > 1 and sys.argv[1] == "--leader": + leader(Path(sys.argv[2]), sys.argv[3], bool(int(sys.argv[4]))) + elif len(sys.argv) > 1 and sys.argv[1] == "--worker": + worker(Path(sys.argv[2]), sys.argv[3], bool(int(sys.argv[4])), + bool(int(sys.argv[5])) if len(sys.argv) > 5 else False) + elif len(sys.argv) > 1 and sys.argv[1] == "--owned-worker": + owned_worker(Path(sys.argv[2]), sys.argv[3]) + else: + main()