From 6b1f1dbcf27a2520cb626b25dedf527d372c2112 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 3 Oct 2026 14:00:14 -0700 Subject: [PATCH 1/3] Test: expose collector writer-authority gap (#174) --- tests/reader_fence_process.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/reader_fence_process.rs b/tests/reader_fence_process.rs index 568ecd40..8c3fec78 100644 --- a/tests/reader_fence_process.rs +++ b/tests/reader_fence_process.rs @@ -8,7 +8,7 @@ pub mod reader; #[path = "segment_filesystem_stage/sandbox.rs"] pub mod sandbox; -use keep::FilesystemWriterLock; +use keep::{FilesystemWriterLock, WriterLockAcquireError}; use rustix::{ fs::{FlockOperation, flock}, io::Errno, @@ -25,6 +25,13 @@ fn reader_death_releases_collection_without_replacing_the_fence() -> Result<(), return reader::serve(); } let store = fixture::migrated("reader-death")?; + assert!( + matches!( + FilesystemWriterLock::try_acquire(store.path()), + Err(WriterLockAcquireError::Busy) + ), + "collector preparation must continuously exclude competing writers" + ); let lock_path = store.path().join("reader.lock"); let before = fs::metadata(&lock_path)?; let mut reader = reader::Reader::spawn( @@ -62,6 +69,13 @@ fn collection_excludes_a_new_snapshot_until_release() -> Result<(), Box Date: Sat, 3 Oct 2026 14:02:34 -0700 Subject: [PATCH 2/3] Fix: retain collector writer authority across fixture handoff (#174) --- CHANGELOG.md | 2 + docs/testing-evidence/reader-fence-process.md | 10 ++++ .../continuous-authority-green.txt | 56 ++++++++++++++++++ .../continuous-authority-red.txt | 59 +++++++++++++++++++ .../hosted-busy-failure.txt | 16 +++++ tests/reader_fence_process.rs | 6 +- tests/reader_fence_process/fixture.rs | 10 +++- 7 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 docs/testing-evidence/reader-fence-process/continuous-authority-green.txt create mode 100644 docs/testing-evidence/reader-fence-process/continuous-authority-red.txt create mode 100644 docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e1158ae..83cf4c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ after its public API and format compatibility policies are established. ## [Unreleased] +- Reader-fence process scenarios retain migration writer authority through collector preparation, removing a release/reacquire gap without changing production lock semantics (#174). + - Linux public snapshot process laws verify reader-death fence release, persistent lock identity and exclusion of new readers during collection with kernel-observed ordering (#113). - Retention model histories now include release and restore, with expected generations and anchor sets derived independently from requested operations rather than copied from publication candidates; exact stale/retry refusals remain checked (#128). diff --git a/docs/testing-evidence/reader-fence-process.md b/docs/testing-evidence/reader-fence-process.md index f44f77cb..46a41abf 100644 --- a/docs/testing-evidence/reader-fence-process.md +++ b/docs/testing-evidence/reader-fence-process.md @@ -35,3 +35,13 @@ The [post-release-admission patch](reader-fence-process/post-release-admission.p Replay each patch independently with `git apply --unidiff-zero` in a fresh copy of the source above. Use `cargo test --all-features --locked --test reader_fence_process reader_death_releases_collection_without_replacing_the_fence -- --exact --nocapture` for the first two experiments; use the same command with `collection_excludes_a_new_snapshot_until_release` for the third. The [restored GREEN receipt](reader-fence-process/restored-green.txt) records the unmodified source tree and both laws passing in debug and release. Final receipt-only successors leave runtime code and test expectations unchanged. Committed logs replace only the isolated container source/build path prefixes with descriptive placeholders and remove trailing empty lines. Patches use zero-context hunks to avoid incidental whitespace in evidence files. The original raw logs, complete experiment variants and traced launcher remain retained by the author. Signal status and channel markers attest execution of the intended schedule; they are not represented as separate product promises. These calibrations do not extend the two fixed schedules, platform coverage, resource enforcement or power-loss claims above. + +## Continuous collector authority correction (#174) + +Change kind: test-isolation bug fix. On the documentation-only successor `e20629852f402e129d889e398014781d20d6ff0b`, [hosted run 37153036962](https://github.com/flyingrobots/keep/actions/runs/37153036962/job/111290654504) failed the collector-exclusion law with bare `Error: Busy`; its [original failure excerpt](reader-fence-process/hosted-busy-failure.txt) is preserved. Earlier passes are not substituted for that failure. The old migrated-store fixture dropped writer authority, after which each law performed a fresh nonblocking acquisition. Concurrent subprocess creation can retain inherited flock descriptions until exec and bridge that gap. This is a source-backed possible schedule, not a demonstrated trace of the hosted failure. One diagnostic strace run with delayed exec passed and establishes no absence of the race. + +The fixture now hands its existing migration authority to the collector without releasing it. Each scenario observes an exact `WriterLockAcquireError::Busy` for a competing public writer acquisition immediately after fixture handoff, before spawning its child. These assertions validate the fixture's continuous-authority contract against the real runtime adapter; they do not independently prove the inherited-descriptor hypothesis or add a new product locking promise. The existing kernel-observed reader schedules, exact contention errno, SIGKILL/reap, inode preservation and post-release admission assertions remain unchanged. No retry, sleep, global serialization, unsafe hook or production unlock behavior was added. + +Regression commit `6b1f1db` on unfixed main `34d70909b0cd93f6b020a59d4d40f43b07971cd8` adds the competing-writer check before the old acquisition. Its [RED receipt](reader-fence-process/continuous-authority-red.txt) executes the collector-exclusion law alone, before any child spawn, and fails the named continuous-exclusion assertion because the old fixture admits a competitor. The [fixed GREEN receipt](reader-fence-process/continuous-authority-green.txt) runs both unchanged reader schedules plus that check in debug and release. Run `cargo test --all-features --locked --test reader_fence_process collection_excludes_a_new_snapshot_until_release -- --exact` to replay the regression; omit the filter and add `--release` to cover both profiles. + +These copied-Docker runs use Rust 1.96.0 on Linux aarch64 and owned ext4 scratch. The deterministic regression calibrates the shared continuous-authority assertion; it does not claim exhaustive fork/exec scheduling or physical power-loss evidence. The original record's resource-enforcement limitations still apply. Retire the new checks if collector preparation no longer requires held writer authority or stronger evidence subsumes the same boundary. Full validation and exact-head independent review are recorded separately in the PR; a focused pass alone is not merge acceptance. diff --git a/docs/testing-evidence/reader-fence-process/continuous-authority-green.txt b/docs/testing-evidence/reader-fence-process/continuous-authority-green.txt new file mode 100644 index 00000000..7961387d --- /dev/null +++ b/docs/testing-evidence/reader-fence-process/continuous-authority-green.txt @@ -0,0 +1,56 @@ + Compiling keep v0.0.0 () + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.36s + Running tests/reader_fence_process.rs (/debug/deps/reader_fence_process-d5b0d43619bf3bf3) + +running 2 tests +test collection_excludes_a_new_snapshot_until_release ... ok +test reader_death_releases_collection_without_replacing_the_fence ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + + Compiling rustix v1.1.4 + Compiling linux-raw-sys v0.12.1 + Compiling io-lifetimes v3.0.1 + Compiling io-lifetimes v2.0.4 + Compiling bitflags v2.13.1 + Compiling proc-macro2 v1.0.107 + Compiling io-extras v0.19.0 + Compiling once_cell v1.21.4 + Compiling cap-primitives v4.0.2 + Compiling find-msvc-tools v0.1.9 + Compiling quote v1.0.47 + Compiling unicode-ident v1.0.24 + Compiling shlex v2.0.1 + Compiling ambient-authority v0.0.2 + Compiling maybe-owned v0.3.4 + Compiling cap-std v4.0.2 + Compiling ipnet v2.12.0 + Compiling cap-fs-ext v4.0.2 + Compiling clap_lex v1.1.0 + Compiling cfg-if v1.0.4 + Compiling anstyle v1.0.14 + Compiling cc v1.3.0 + Compiling libc v0.2.186 + Compiling arrayref v0.3.9 + Compiling arrayvec v0.7.8 + Compiling constant_time_eq v0.4.2 + Compiling regex-lite v0.1.9 + Compiling clap_builder v4.6.2 + Compiling condtype v1.3.0 + Compiling allocation-counter v0.8.1 + Compiling syn v2.0.119 + Compiling blake3 v1.8.5 + Compiling clap v4.6.4 + Compiling rustix-linux-procfs v0.1.1 + Compiling fs-set-times v0.20.3 + Compiling divan-macros v0.1.21 + Compiling keep v0.0.0 () + Compiling divan v0.1.21 + Finished `release` profile [optimized] target(s) in 3.70s + Running tests/reader_fence_process.rs (/release/deps/reader_fence_process-2a35bc062b89d081) + +running 2 tests +test collection_excludes_a_new_snapshot_until_release ... ok +test reader_death_releases_collection_without_replacing_the_fence ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s diff --git a/docs/testing-evidence/reader-fence-process/continuous-authority-red.txt b/docs/testing-evidence/reader-fence-process/continuous-authority-red.txt new file mode 100644 index 00000000..c4ddd9ea --- /dev/null +++ b/docs/testing-evidence/reader-fence-process/continuous-authority-red.txt @@ -0,0 +1,59 @@ + Compiling rustix v1.1.4 + Compiling linux-raw-sys v0.12.1 + Compiling io-lifetimes v3.0.1 + Compiling io-lifetimes v2.0.4 + Compiling bitflags v2.13.1 + Compiling io-extras v0.19.0 + Compiling proc-macro2 v1.0.107 + Compiling quote v1.0.47 + Compiling shlex v2.0.1 + Compiling unicode-ident v1.0.24 + Compiling once_cell v1.21.4 + Compiling cap-primitives v4.0.2 + Compiling find-msvc-tools v0.1.9 + Compiling ambient-authority v0.0.2 + Compiling ipnet v2.12.0 + Compiling cap-std v4.0.2 + Compiling maybe-owned v0.3.4 + Compiling cap-fs-ext v4.0.2 + Compiling cfg-if v1.0.4 + Compiling libc v0.2.186 + Compiling clap_lex v1.1.0 + Compiling cc v1.3.0 + Compiling anstyle v1.0.14 + Compiling arrayvec v0.7.8 + Compiling constant_time_eq v0.4.2 + Compiling arrayref v0.3.9 + Compiling clap_builder v4.6.2 + Compiling condtype v1.3.0 + Compiling regex-lite v0.1.9 + Compiling allocation-counter v0.8.1 + Compiling syn v2.0.119 + Compiling blake3 v1.8.5 + Compiling clap v4.6.4 + Compiling fs-set-times v0.20.3 + Compiling rustix-linux-procfs v0.1.1 + Compiling divan-macros v0.1.21 + Compiling keep v0.0.0 () + Compiling divan v0.1.21 + Finished `test` profile [unoptimized + debuginfo] target(s) in 3.33s + Running tests/reader_fence_process.rs (/debug/deps/reader_fence_process-d5b0d43619bf3bf3) + +running 1 test +test collection_excludes_a_new_snapshot_until_release ... FAILED + +failures: + +---- collection_excludes_a_new_snapshot_until_release stdout ---- + +thread 'collection_excludes_a_new_snapshot_until_release' (2000794) panicked at tests/reader_fence_process.rs:72:5: +collector preparation must continuously exclude competing writers +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + collection_excludes_a_new_snapshot_until_release + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.04s + +error: test failed, to rerun pass `--test reader_fence_process` diff --git a/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt b/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt new file mode 100644 index 00000000..0c305cb1 --- /dev/null +++ b/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt @@ -0,0 +1,16 @@ +test collection_excludes_a_new_snapshot_until_release ... FAILED +2026-10-03T20:54:11.0422983Z test reader_death_releases_collection_without_replacing_the_fence ... ok +2026-10-03T20:54:11.0423520Z +2026-10-03T20:54:11.0423612Z failures: +2026-10-03T20:54:11.0423714Z +2026-10-03T20:54:11.0423957Z ---- collection_excludes_a_new_snapshot_until_release stdout ---- +2026-10-03T20:54:11.0424328Z Error: Busy +2026-10-03T20:54:11.0424451Z +2026-10-03T20:54:11.0424471Z +2026-10-03T20:54:11.0424558Z failures: +2026-10-03T20:54:11.0424793Z collection_excludes_a_new_snapshot_until_release +2026-10-03T20:54:11.0425083Z +2026-10-03T20:54:11.0425373Z test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +2026-10-03T20:54:11.0425777Z +2026-10-03T20:54:11.0426024Z error: test failed, to rerun pass `-p keep --test reader_fence_process` +2026-10-03T20:54:11.0455695Z ##[error]Process completed with exit code 101. diff --git a/tests/reader_fence_process.rs b/tests/reader_fence_process.rs index 8c3fec78..2e85ca3b 100644 --- a/tests/reader_fence_process.rs +++ b/tests/reader_fence_process.rs @@ -24,7 +24,7 @@ fn reader_death_releases_collection_without_replacing_the_fence() -> Result<(), if reader::is_child() { return reader::serve(); } - let store = fixture::migrated("reader-death")?; + let (store, writer) = fixture::migrated("reader-death")?; assert!( matches!( FilesystemWriterLock::try_acquire(store.path()), @@ -39,7 +39,6 @@ fn reader_death_releases_collection_without_replacing_the_fence() -> Result<(), "reader_death_releases_collection_without_replacing_the_fence", )?; reader.await_snapshot()?; - let writer = FilesystemWriterLock::try_acquire(store.path())?; let collector = fs::File::open(&lock_path)?; assert_eq!( flock(&collector, FlockOperation::NonBlockingLockExclusive), @@ -68,7 +67,7 @@ fn collection_excludes_a_new_snapshot_until_release() -> Result<(), Box Result<(), Box Result> { +pub(super) fn migrated( + name: &str, +) -> Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { let sandbox = TestDirectory::create(name)?; let mut storage = RepositoryInitializationStorage::admit_unchecked(sandbox.path())?; let _initialized = initialize_store(&mut storage)?; @@ -33,8 +35,10 @@ pub(super) fn migrated(name: &str) -> Result> { )?; let intent = migration.observe_intent()?; let _receipt = execute_store_migration(&mut migration, &intent)?; - drop(migration); - Ok(sandbox) + // Keep writer authority continuous while the caller arranges collection. + // A concurrent spawn may inherit lock descriptions until exec, so dropping + // and immediately reacquiring cannot assume all holders have disappeared. + Ok((sandbox, migration)) } fn decode(hex: &str) -> Result, Box> { From e1300cf4a866bd34cba4ebf7dd50233ca264dada Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 3 Oct 2026 14:05:46 -0700 Subject: [PATCH 3/3] Docs: normalize hosted receipt line endings (#174) --- docs/testing-evidence/reader-fence-process.md | 2 +- .../reader-fence-process/hosted-busy-failure.txt | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/testing-evidence/reader-fence-process.md b/docs/testing-evidence/reader-fence-process.md index 46a41abf..d2ef0786 100644 --- a/docs/testing-evidence/reader-fence-process.md +++ b/docs/testing-evidence/reader-fence-process.md @@ -38,7 +38,7 @@ Committed logs replace only the isolated container source/build path prefixes wi ## Continuous collector authority correction (#174) -Change kind: test-isolation bug fix. On the documentation-only successor `e20629852f402e129d889e398014781d20d6ff0b`, [hosted run 37153036962](https://github.com/flyingrobots/keep/actions/runs/37153036962/job/111290654504) failed the collector-exclusion law with bare `Error: Busy`; its [original failure excerpt](reader-fence-process/hosted-busy-failure.txt) is preserved. Earlier passes are not substituted for that failure. The old migrated-store fixture dropped writer authority, after which each law performed a fresh nonblocking acquisition. Concurrent subprocess creation can retain inherited flock descriptions until exec and bridge that gap. This is a source-backed possible schedule, not a demonstrated trace of the hosted failure. One diagnostic strace run with delayed exec passed and establishes no absence of the race. +Change kind: test-isolation bug fix. On the documentation-only successor `e20629852f402e129d889e398014781d20d6ff0b`, [hosted run 37153036962](https://github.com/flyingrobots/keep/actions/runs/37153036962/job/111290654504) failed the collector-exclusion law with bare `Error: Busy`; its [failure excerpt](reader-fence-process/hosted-busy-failure.txt) preserves the original diagnostics and timestamps, trimming only line-end whitespace. The unnormalized raw log is retained separately. Earlier passes are not substituted for that failure. The old migrated-store fixture dropped writer authority, after which each law performed a fresh nonblocking acquisition. Concurrent subprocess creation can retain inherited flock descriptions until exec and bridge that gap. This is a source-backed possible schedule, not a demonstrated trace of the hosted failure. One diagnostic strace run with delayed exec passed and establishes no absence of the race. The fixture now hands its existing migration authority to the collector without releasing it. Each scenario observes an exact `WriterLockAcquireError::Busy` for a competing public writer acquisition immediately after fixture handoff, before spawning its child. These assertions validate the fixture's continuous-authority contract against the real runtime adapter; they do not independently prove the inherited-descriptor hypothesis or add a new product locking promise. The existing kernel-observed reader schedules, exact contention errno, SIGKILL/reap, inode preservation and post-release admission assertions remain unchanged. No retry, sleep, global serialization, unsafe hook or production unlock behavior was added. diff --git a/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt b/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt index 0c305cb1..f990db78 100644 --- a/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt +++ b/docs/testing-evidence/reader-fence-process/hosted-busy-failure.txt @@ -1,16 +1,16 @@ test collection_excludes_a_new_snapshot_until_release ... FAILED 2026-10-03T20:54:11.0422983Z test reader_death_releases_collection_without_replacing_the_fence ... ok -2026-10-03T20:54:11.0423520Z +2026-10-03T20:54:11.0423520Z 2026-10-03T20:54:11.0423612Z failures: -2026-10-03T20:54:11.0423714Z +2026-10-03T20:54:11.0423714Z 2026-10-03T20:54:11.0423957Z ---- collection_excludes_a_new_snapshot_until_release stdout ---- 2026-10-03T20:54:11.0424328Z Error: Busy -2026-10-03T20:54:11.0424451Z -2026-10-03T20:54:11.0424471Z +2026-10-03T20:54:11.0424451Z +2026-10-03T20:54:11.0424471Z 2026-10-03T20:54:11.0424558Z failures: 2026-10-03T20:54:11.0424793Z collection_excludes_a_new_snapshot_until_release -2026-10-03T20:54:11.0425083Z +2026-10-03T20:54:11.0425083Z 2026-10-03T20:54:11.0425373Z test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s -2026-10-03T20:54:11.0425777Z +2026-10-03T20:54:11.0425777Z 2026-10-03T20:54:11.0426024Z error: test failed, to rerun pass `-p keep --test reader_fence_process` 2026-10-03T20:54:11.0455695Z ##[error]Process completed with exit code 101.