diff --git a/docs/research/sparse-baseline-windows-argv.md b/docs/research/sparse-baseline-windows-argv.md new file mode 100644 index 0000000000..07a8bcf51e --- /dev/null +++ b/docs/research/sparse-baseline-windows-argv.md @@ -0,0 +1,128 @@ +# Large selected inventories in Windows sparse checkouts + +The selected-source preflight could reject a valid sparse checkout on Windows +because it passed every omitted input to Git in one process command. This repair +batches calls to the existing sparse validator. It preserves the file checks, +sorted omission list, materialization advice, schema routing and exit codes. + +## Reproduced failure + +At canonical main `cef4b52f2a59009865bf8eabde1be3f159063703`, the selected +inventory contains 523 distinct paths. A cone checkout containing `scripts`, +`docs` and `certificates` omitted 503 of them. The resulting `git ls-files` +command was 38,351 characters long. On native Windows with CPython 3.12.0 and +Git 2.47.1.windows.1 it failed with `FileNotFoundError [WinError 206]`. +The validator caught the launch error and then reported +`Invalid selected file: notes/batched-bit-rows.tex`, although that file had a +regular stage-zero skip-worktree entry matching HEAD. Checking it alone passed. + +Microsoft documents the process command limit as 32,767 characters including +the terminating null in +[CreateProcessW](https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw). +The new batching uses a conservative 16,000 UTF-16-unit path-argument budget, +including Windows quoting and separators, leaving room for Git's fixed arguments. +An individually unlaunchable path remains an error; batching addresses the +aggregate inventory size, not arbitrary filesystem path limits. + +Every batch still calls `choose_research_task.check_catalog_files`. Ordinary +deletions, nonregular entries, changed index blobs and Git failures retain that +validator's rejection behavior. A later failure raises before any omission +report is printed. As before, the checkout must remain stable during inspection; +this is not an atomic snapshot or a source-hash check. + +## Evidence and reproduction + +The new real-Git fixture contains 600 omitted files with spaces, literal +pathspec metacharacters and a supplementary Unicode character. Its unbatched +command exceeds the Windows limit. The fixture checks complete path coverage +in both index and HEAD inventories, successful CLI materialization and a final +zero-omission report. Controls change the final input's index blob, turn it into +an ordinary deletion, and fail a later Git inventory call. The command-size +failure is also simulated on other platforms while actual Git supplies the data. + +Before the fix, the five new tests all failed on Windows (four failures and one +error), including the unmocked CLI. After the fix, all 14 sparse-baseline tests +passed normally and under `-O` on Windows; the same 14 passed on Ubuntu/WSL. +The existing small cone/non-cone, legacy/explicit manifest, omitted metadata and +maintenance-cleanup fixtures remain unchanged. + +After materializing the current selected inputs, the preflight returned +`NO_SPARSE_SELECTED_OMISSIONS` and the unchanged selected baseline returned +`PASS_RESEARCH_BASELINE_BINDINGS_ONLY` on Windows. This authenticates the +selected metadata closure; the full mathematical cold replay was not run. + +All 6,136 tracked top-level `research/*/SOURCES.json` and `VERIFY.json` files +were read and their working bytes compared with their HEAD Git blob IDs. None +has an exact-path live pin, provenance pin or declared replay input for +`scripts/research_baseline_sparse.py`. Nested manifests and undeclared runtime +reads are outside this dependency check. The pinned chooser and baseline +implementations retain their bytes. + +The broader Windows helper command (`PYTHONUTF8=1`, pattern +`test_research_*.py`) ran 362 tests: 322 passed, 22 skipped, and 18 had setup +errors. Twelve fixtures require newline-containing filenames unsupported by +Windows; six initially lacked the sparsely omitted descendant-bytecode fixture. +After materializing that package and its historical source objects, its native +rerun passed three methods but three still failed at historical-source resolution +through the fixture's alternate object store. All 18 affected methods passed on +Ubuntu/WSL (the six-case `test_research_overlap_local.py`, +`test_research_plan_explanations.py` and `test_research_descendant_source_roles.py` +modules). Those unchanged fixtures and their pins are outside this repair. +No clean full native Windows run or complete local Linux suite is claimed. + +```sh +python -W error -B -m unittest discover -s tests -p 'test_research_baseline_sparse*.py' -v +python -W error -O -B -m unittest discover -s tests -p 'test_research_baseline_sparse*.py' -v +python -B scripts/research_baseline_sparse.py +``` + +Exit 2 means valid sparse omissions were found; exit 0 means no input is omitted. +Neither outcome authenticates the input bytes or proves a mathematical claim. +The subsequent `research_selected_baseline.py` and selected cold-replay command +retain their separate obligations. + +## Task, overlap and attribution + +Card: `portable-replay`, priority revision 7, chooser nonce +`1568f02fbdb6dc3102dbdeab4f3c4e6e`, scope key `831cc1a3c8f9ff01b197`. +Scope refined to the specific native Windows launch failure encountered during +selected-baseline orientation. The bounded success condition is a working large +inventory CLI with complete coverage and retained rejection controls. No +scientific package or selected-frontier file is changed; no new kappa is claimed. + +The [canonical claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/4458#discussioncomment-19136447) +records the base, method and limits. Entry overlap covered 304 open PR bodies +and 4,003 Discussion records at 2026-10-23 02:22/02:23 UTC, with complete reported +pagination. Sparse local-note coverage was incomplete at entry. Targeted live +and closed-PR searches found the following distinct predecessors; these checks +do not establish novelty against every diff or unpublished investigation. + +The submission refresh at 02:38/02:40 UTC covered 308 open PR bodies, 4,007 +Discussion records and all 6,601 tracked in-scope local notes, with complete +reported coverage and no warnings. The exact command-size search found this +claim and the earlier fixture-maintenance handoff. Canonical main was fetched +again before submission and remained the same base commit. + +- Douglas Colkitt and the credited Claude agent's preflight, published by + Mister-Meeseeks in [#3725](https://github.com/CrocSwap/integer-mult-bounds/pull/3725). +- Arham-Tahir64's unchanged sparse/HEAD contract in + [#3102](https://github.com/CrocSwap/integer-mult-bounds/pull/3102). +- lee-phys's inventory-format and schema-routing repair in + [#4631](https://github.com/CrocSwap/integer-mult-bounds/pull/4631). +- Clear1nsight's fixture-maintenance repair in + [#4857](https://github.com/CrocSwap/integer-mult-bounds/pull/4857). +- Mihir Srivastava's productive-helper baseline and generic startup guidance, + [#4463](https://github.com/CrocSwap/integer-mult-bounds/pull/4463) and + [#4906](https://github.com/CrocSwap/integer-mult-bounds/pull/4906). + +**Open next question:** the preflight's materialization advice still emits one +directory command. If a future selected closure has enough distinct directories +to exceed a process limit, reproduce that directory-level case and provide +bounded materialization advice without changing the authenticated source set. +The present selected closure does not exhibit that second failure. The generic +validator itself also retains its existing behavior for callers other than this +preflight; no claim is made about all large catalogues or arbitrary Git tools. + +Prepared by Mihir Srivastava with substantial OpenAI Codex assistance; no +subagents or independent human review. New work is Apache-2.0. Existing source +notices and frozen scientific evidence retain their attribution. diff --git a/scripts/research_baseline_sparse.py b/scripts/research_baseline_sparse.py index 4e4c649798..c92a7b947a 100644 --- a/scripts/research_baseline_sparse.py +++ b/scripts/research_baseline_sparse.py @@ -34,6 +34,9 @@ ROOT = Path(__file__).resolve().parents[1] SELECTED = "certificates/selected-result.json" +# Leave ample room for Git's fixed arguments under CreateProcessW's 32,767 +# UTF-16-unit limit. Bound quoted path arguments, not their count or UTF-8 size. +PATHSPEC_BUDGET = 16_000 def selected_json(name, root): @@ -64,7 +67,20 @@ def selected_inputs(record, root=ROOT): def sparse_omissions(record, root=ROOT): """Selected inputs absent only because this is a sparse checkout.""" names = selected_inputs(record, root) - return check_catalog_files(root, [(name, "Invalid selected file: " + name) for name in names]) + omitted, batch, units = [], [], 0 + for name in names: + quoted = subprocess.list2cmdline([name]) + size = len(quoted.encode("utf-16-le", errors="surrogatepass")) // 2 + 1 + if batch and units + size > PATHSPEC_BUDGET: + omitted.extend(check_catalog_files(root, batch)) + batch, units = [], 0 + batch.append((name, "Invalid selected file: " + name)) + units += size + if batch: + omitted.extend(check_catalog_files(root, batch)) + # Validation remains all-or-nothing: a later invalid entry or Git failure + # raises before report() can publish any earlier batch's omissions. + return omitted def materialize_command(omitted): diff --git a/tests/test_research_baseline_sparse_large.py b/tests/test_research_baseline_sparse_large.py new file mode 100644 index 0000000000..03ae5249db --- /dev/null +++ b/tests/test_research_baseline_sparse_large.py @@ -0,0 +1,132 @@ +"""Large sparse inventories must fit Windows argv without weakening validation. + +Mihir Srivastava, with substantial OpenAI Codex assistance. Apache-2.0. +""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import research_baseline_sparse as sparse +sys.path.pop(0) + + +class LargeSparseBaselineTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + # Spaces exercise Windows quoting; brackets exercise literal pathspecs; + # the supplementary character occupies two UTF-16 code units. + self.names = [f"research/omitted/{i:04d} [payload] " + "long-name-" * 7 + "\U0001f9ea.txt" + for i in range(600)] + self.record = { + "certificate": "certificates/claim.json", + "source_manifest": "certificates/source.json", + "proof": "docs/proof.md", "review": "docs/proof.md", + "validation": "docs/proof.md", "proof_supplements": {}, + } + files = { + "certificates/selected-result.json": json.dumps(self.record), + "certificates/claim.json": json.dumps({"schema": sparse.PRODUCTIVE_SCHEMA}), + "certificates/source.json": json.dumps({ + "format": "repository-files-sha256-v1", + "files": {name: "0" for name in self.names}, + }), + "docs/proof.md": "fixture\n", + **{name: "input\n" for name in self.names}, + } + for name, content in files.items(): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + self.git("init", "-q") + self.git("-c", "core.autocrlf=false", "add", ".") + self.git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + "-c", "maintenance.auto=false", "-c", "gc.auto=0", + "-c", "core.hooksPath=/dev/null", "commit", "-qm", "fixture") + self.git("sparse-checkout", "set", "--cone", "certificates", "docs") + + def git(self, *args, **kwargs): + return subprocess.run(["git", *args], cwd=self.root, check=True, + capture_output=True, **kwargs).stdout + + def test_complete_inventory_with_windows_launch_limit(self): + original_run = subprocess.run + inventories = [] + + def windows_limit(argv, **kwargs): + if argv[:2] == ["git", "--literal-pathspecs"]: + units = len(subprocess.list2cmdline(argv).encode("utf-16-le")) // 2 + if units >= 32767: + raise OSError(206, "simulated Windows command line limit") + inventories.append((argv[2], argv[argv.index("--") + 1:])) + return original_run(argv, **kwargs) + + unbatched = ["git", "--literal-pathspecs", "ls-files", "-t", "--stage", "-z", + "--", *self.names] + self.assertGreater(len(subprocess.list2cmdline(unbatched)), 32767) + with mock.patch.object(subprocess, "run", side_effect=windows_limit): + omitted = sparse.sparse_omissions(self.record, self.root) + self.assertEqual(omitted, self.names) + for command in ("ls-files", "ls-tree"): + batches = [paths for kind, paths in inventories if kind == command] + self.assertGreater(len(batches), 1) + self.assertEqual([name for paths in batches for name in paths], self.names) + + def test_real_cli_materializes_every_input(self): + command = [sys.executable, "-B", str(ROOT / "scripts/research_baseline_sparse.py"), + "--root", str(self.root)] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 2, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["omitted_files"], len(self.names)) + self.assertEqual(report["first_omitted"], self.names[:5]) + self.assertEqual(report["next"], ["python3", "-B", "scripts/research_selected_baseline.py"]) + self.git(*report["materialize"][1:]) + self.assertTrue(all((self.root / name).is_file() for name in self.names)) + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)["status"], "NO_SPARSE_SELECTED_OMISSIONS") + + def test_late_ordinary_deletion_rejected(self): + name = self.names[-1] + self.git("update-index", "--no-skip-worktree", "--", name) + with self.assertRaises(ValueError) as failure: + sparse.sparse_omissions(self.record, self.root) + self.assertEqual(str(failure.exception), "Invalid selected file: " + name) + + def test_late_staged_blob_drift_rejected(self): + name = self.names[-1] + blob = self.git("hash-object", "-w", "--stdin", input=b"changed\n").decode().strip() + self.git("update-index", "--cacheinfo", "100644," + blob + "," + name) + self.git("update-index", "--skip-worktree", "--", name) + with self.assertRaises(ValueError) as failure: + sparse.sparse_omissions(self.record, self.root) + self.assertEqual(str(failure.exception), "Invalid selected file: " + name) + + def test_late_git_failure_does_not_publish_partial_report(self): + original_run = subprocess.run + calls = 0 + + def fail_second_inventory(argv, **kwargs): + nonlocal calls + if argv[:3] == ["git", "--literal-pathspecs", "ls-files"]: + calls += 1 + if calls == 2: + raise subprocess.CalledProcessError(1, argv, stderr="injected failure") + return original_run(argv, **kwargs) + + with mock.patch.object(subprocess, "run", side_effect=fail_second_inventory): + with self.assertRaises(ValueError): + sparse.report(self.record, self.root) + self.assertEqual(calls, 2, "the control must reach a later inventory batch") + + +if __name__ == "__main__": + unittest.main()