From 8d0af53f01b9146843a4f292634e5ee334482df6 Mon Sep 17 00:00:00 2001 From: "zackary.l.jackson" Date: Sun, 4 Oct 2026 19:01:44 +0000 Subject: [PATCH 1/6] fix(private-fs): admit a free lock after its admission deadline Bounded lock admission counted the caller's own latency against the deadline: it refused before its first try_lock once the deadline had passed, and it released a lock it had just taken if the deadline passed during the attempt. A thread descheduled past its budget therefore timed out on a free lock. The hook spool's settlement reacquires its writer lease with the lease duration as its budget, which is 100us in the spool test fixture, so a short stall on a loaded runner failed single-writer spool tests with AdmissionTimedOut (#3081). Admission now always attempts the lock once and times out only when a holder outlasts the deadline. The delivery spool's try_lock-first special case, which existed to get that behavior, is deleted. Closes #3081 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/tracedecay-hooks/src/delivery_spool.rs | 31 ++------ crates/tracedecay-hooks/src/spool/tests.rs | 44 +++++++++-- .../src/lock_admission.rs | 74 +++++++++++++++---- 3 files changed, 106 insertions(+), 43 deletions(-) diff --git a/crates/tracedecay-hooks/src/delivery_spool.rs b/crates/tracedecay-hooks/src/delivery_spool.rs index 1f767094d6..6a416f349b 100644 --- a/crates/tracedecay-hooks/src/delivery_spool.rs +++ b/crates/tracedecay-hooks/src/delivery_spool.rs @@ -276,23 +276,10 @@ impl HookDeliveryReceiptSpoolV1 { let root = root.into(); ensure_root(&root)?; let lock = open_lock_file(&root, LOCK_FILE)?; - let try_lock_result = { - let _span = tracing::trace_span!("hooks.delivery.lock.try_lock").entered(); - lock.try_lock() - }; - match try_lock_result { - Ok(()) => {} - Err(std::fs::TryLockError::WouldBlock) => { - if wait_budget.is_zero() { - return Err(HookDeliverySpoolError::Busy); - } - lock_until(&lock, Instant::now() + wait_budget).map_err(|error| match error { - LockAdmissionError::TimedOut => HookDeliverySpoolError::Busy, - LockAdmissionError::Io(_) => HookDeliverySpoolError::Io, - })?; - } - Err(std::fs::TryLockError::Error(_)) => return Err(HookDeliverySpoolError::Io), - } + lock_until(&lock, Instant::now() + wait_budget).map_err(|error| match error { + LockAdmissionError::TimedOut => HookDeliverySpoolError::Busy, + LockAdmissionError::Io(_) => HookDeliverySpoolError::Io, + })?; let spool = Self { root, _lock: FileLease::held(lock, "hooks.delivery.writer"), @@ -834,13 +821,9 @@ mod tests { HookDeliveryReceiptSpoolV1::open(&root.0, Duration::ZERO).unwrap_err(), HookDeliverySpoolError::Busy ); - // An exhausted budget never admits, even when the lease is free. - assert_eq!( - HookDeliveryReceiptWriterV1::open_within(&root.0, Duration::ZERO).unwrap_err(), - HookDeliverySpoolError::AdmissionTimedOut - ); - let writer = - HookDeliveryReceiptWriterV1::open_within(&root.0, Duration::from_millis(20)).unwrap(); + // A spent budget still takes the free staging lease; only the held + // publish lock is out of reach, so the receipt stages. + let writer = HookDeliveryReceiptWriterV1::open_within(&root.0, Duration::ZERO).unwrap(); assert_eq!( writer.retain(&receipt()).unwrap(), HookDeliveryRetentionV1::Staged diff --git a/crates/tracedecay-hooks/src/spool/tests.rs b/crates/tracedecay-hooks/src/spool/tests.rs index 4334e7bba2..efdbe49b21 100644 --- a/crates/tracedecay-hooks/src/spool/tests.rs +++ b/crates/tracedecay-hooks/src/spool/tests.rs @@ -1699,6 +1699,44 @@ fn settling_and_reclaiming_hold_the_writer_lease_across_no_durability_barrier() assert_eq!(fs::metadata(records_path(&root.0)).unwrap().len(), 0); } +/// Bounded admission waits only on contention. A writer whose budget is +/// already spent, like a callback or a drain descheduled past its deadline, +/// still takes a free lease, including the lease a settlement reacquires +/// after its barriers; a held lease refuses it without waiting. +#[test] +fn a_writer_whose_budget_is_spent_takes_a_free_lease_but_never_a_held_one() { + let root = TestDir::new("spent-budget"); + let (owner, _) = HookSpoolV1::open(&root.0, config(), UtcMicros(10)).unwrap(); + assert_eq!( + HookSpoolV1::open_within(&root.0, config(), UtcMicros(11), Duration::ZERO).unwrap_err(), + HookSpoolError::AdmissionTimedOut + ); + drop(owner); + + let (mut admitted, _) = + HookSpoolV1::open_within(&root.0, config(), UtcMicros(11), Duration::ZERO) + .expect("a free lease admits a writer with no budget left"); + let record = admitted + .append(envelope(1, 9), &binding(), UtcMicros(11)) + .unwrap(); + assert!( + admitted + .acknowledge( + HookSpoolAckV1 { + sequence: record.sequence, + receipt_id: [41; 16], + disposition: HookSpoolAckDispositionV1::Committed, + }, + UtcMicros(11), + ) + .unwrap() + ); + drop(admitted); + let (_, report) = HookSpoolV1::open(&root.0, config(), UtcMicros(12)).unwrap(); + assert_eq!(report.pending_records, 0); + assert_eq!(report.next_sequence, 2); +} + #[test] fn bounded_writer_admission_preserves_failfast_and_times_out_without_mutation() { let root = TestDir::new("bounded-admission"); @@ -1720,12 +1758,6 @@ fn bounded_writer_admission_preserves_failfast_and_times_out_without_mutation() ); assert_eq!(fs::read(meta_path(&root.0)).unwrap(), before); drop(owner); - // An exhausted budget never admits, even when the lease is free. - assert_eq!( - HookSpoolV1::open_within(&root.0, config(), UtcMicros(11), std::time::Duration::ZERO) - .unwrap_err(), - HookSpoolError::AdmissionTimedOut - ); let (mut admitted, _) = HookSpoolV1::open_within( &root.0, config(), diff --git a/crates/tracedecay-private-fs/src/lock_admission.rs b/crates/tracedecay-private-fs/src/lock_admission.rs index 84400bede9..64280aee7c 100644 --- a/crates/tracedecay-private-fs/src/lock_admission.rs +++ b/crates/tracedecay-private-fs/src/lock_admission.rs @@ -12,37 +12,30 @@ pub enum LockAdmissionError { // Avoid spinning while the admitted writer completes durable filesystem work. const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(1); -/// Exclusive-locks `file`, waiting until `deadline`. -/// A lock taken after the deadline is released and the call times out. +/// Exclusive-locks `file`, waiting until `deadline` for any other holder. #[tracing::instrument(name = "private_fs.lock.admission", level = "trace", skip_all)] pub fn lock_until(file: &File, deadline: Instant) -> Result<(), LockAdmissionError> { admit_until(file, deadline, File::try_lock) } /// Shared-locks `file`, waiting until `deadline` for any exclusive holder. -/// A lock taken after the deadline is released and the call times out. #[tracing::instrument(name = "private_fs.lock.shared_admission", level = "trace", skip_all)] pub fn lock_shared_until(file: &File, deadline: Instant) -> Result<(), LockAdmissionError> { admit_until(file, deadline, File::try_lock_shared) } +/// The deadline bounds waiting on contention, not the caller's own latency: +/// the lock is always attempted once, so a caller descheduled past its +/// deadline still takes a free lock, and only a holder that outlasts the +/// deadline times out. fn admit_until( file: &File, deadline: Instant, try_lock: impl Fn(&File) -> Result<(), std::fs::TryLockError>, ) -> Result<(), LockAdmissionError> { loop { - if Instant::now() >= deadline { - return Err(LockAdmissionError::TimedOut); - } match try_lock(file) { - Ok(()) => { - if Instant::now() >= deadline { - file.unlock().map_err(LockAdmissionError::Io)?; - return Err(LockAdmissionError::TimedOut); - } - return Ok(()); - } + Ok(()) => return Ok(()), Err(std::fs::TryLockError::WouldBlock) => { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { @@ -54,3 +47,58 @@ fn admit_until( } } } + +#[cfg(test)] +mod tests { + use std::fs::OpenOptions; + use std::path::Path; + + use super::*; + + fn open(path: &Path) -> File { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path) + .unwrap() + } + + #[test] + fn a_free_lock_is_admitted_after_the_deadline_has_passed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("admission.lock"); + let elapsed = Instant::now(); + + let exclusive = open(&path); + assert!(lock_until(&exclusive, elapsed).is_ok()); + exclusive.unlock().unwrap(); + + let shared = open(&path); + assert!(lock_shared_until(&shared, elapsed).is_ok()); + let second_reader = open(&path); + assert!(lock_shared_until(&second_reader, elapsed).is_ok()); + } + + #[test] + fn a_held_lock_times_out_once_the_deadline_passes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("admission.lock"); + let holder = open(&path); + holder.lock().unwrap(); + let waiter = open(&path); + + assert!(matches!( + lock_until(&waiter, Instant::now()), + Err(LockAdmissionError::TimedOut) + )); + assert!(matches!( + lock_shared_until(&waiter, Instant::now() + Duration::from_millis(20)), + Err(LockAdmissionError::TimedOut) + )); + + holder.unlock().unwrap(); + assert!(lock_until(&waiter, Instant::now()).is_ok()); + } +} From 814075edd1bb75e8b0a9d50c13b3e6515c6fa91c Mon Sep 17 00:00:00 2001 From: "zackary.l.jackson" Date: Sun, 4 Oct 2026 19:05:06 +0000 Subject: [PATCH 2/6] fix(private-fs): time out a contended wait at its deadline A waiter that saw WouldBlock slept up to the deadline and then retried unconditionally, so a holder releasing after the deadline still admitted it. Check the deadline after each contention sleep; the first attempt stays unconditional so a free lock is still taken past the deadline. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/lock_admission.rs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/lock_admission.rs b/crates/tracedecay-private-fs/src/lock_admission.rs index 64280aee7c..eeeff20b0a 100644 --- a/crates/tracedecay-private-fs/src/lock_admission.rs +++ b/crates/tracedecay-private-fs/src/lock_admission.rs @@ -26,8 +26,8 @@ pub fn lock_shared_until(file: &File, deadline: Instant) -> Result<(), LockAdmis /// The deadline bounds waiting on contention, not the caller's own latency: /// the lock is always attempted once, so a caller descheduled past its -/// deadline still takes a free lock, and only a holder that outlasts the -/// deadline times out. +/// deadline still takes a free lock. Once contended, every retry happens +/// before the deadline, so a holder that outlasts the deadline times out. fn admit_until( file: &File, deadline: Instant, @@ -42,6 +42,9 @@ fn admit_until( return Err(LockAdmissionError::TimedOut); } std::thread::sleep(remaining.min(LOCK_POLL_INTERVAL)); + if Instant::now() >= deadline { + return Err(LockAdmissionError::TimedOut); + } } Err(std::fs::TryLockError::Error(error)) => return Err(LockAdmissionError::Io(error)), } @@ -101,4 +104,23 @@ mod tests { holder.unlock().unwrap(); assert!(lock_until(&waiter, Instant::now()).is_ok()); } + + #[test] + fn a_holder_releasing_after_the_deadline_never_admits_a_contended_waiter() { + let dir = tempfile::tempdir().unwrap(); + let file = open(&dir.path().join("admission.lock")); + let deadline = Instant::now() + Duration::from_millis(5); + let released_after_deadline = |_: &File| { + if Instant::now() < deadline { + Err(std::fs::TryLockError::WouldBlock) + } else { + Ok(()) + } + }; + + assert!(matches!( + admit_until(&file, deadline, released_after_deadline), + Err(LockAdmissionError::TimedOut) + )); + } } From 927c3927e6d0d564b3fb8b66ef3616fa2a6a2275 Mon Sep 17 00:00:00 2001 From: "zackary.l.jackson" Date: Mon, 5 Oct 2026 05:47:16 +0000 Subject: [PATCH 3/6] style(agent-hosts): inline context-scout outcome guards Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/agents/context_scout/owner.rs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs index e06de7ab96..7cd16aea08 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/owner.rs @@ -1012,26 +1012,20 @@ impl ProjectContextScoutOwnerV1 { let retired = match (&mutation, &receipt.result) { ( ContextScoutPublicMutationV1::Cancel { work }, - ContextScoutMutationResultV1::Cancel(outcome), - ) if matches!( - outcome, - ContextScoutDurableStoreOutcomeV1::Stored - | ContextScoutDurableStoreOutcomeV1::Duplicate - ) => - { - Some((*work, None)) - } + ContextScoutMutationResultV1::Cancel( + ContextScoutDurableStoreOutcomeV1::Stored + | ContextScoutDurableStoreOutcomeV1::Duplicate, + ), + ) => Some((*work, None)), ( ContextScoutPublicMutationV1::Delivery { work, .. }, - ContextScoutMutationResultV1::Delivery { outcome, receipt }, - ) if matches!( - outcome, - ContextScoutDurableStoreOutcomeV1::Stored - | ContextScoutDurableStoreOutcomeV1::Duplicate - ) => - { - Some((*work, Some(receipt.outcome))) - } + ContextScoutMutationResultV1::Delivery { + outcome: + ContextScoutDurableStoreOutcomeV1::Stored + | ContextScoutDurableStoreOutcomeV1::Duplicate, + receipt, + }, + ) => Some((*work, Some(receipt.outcome))), _ => None, }; if let Some((work, delivery)) = retired From 173623b93c8a7d93b4c94ac45e771172e1e95276 Mon Sep 17 00:00:00 2001 From: "zackary.l.jackson" Date: Mon, 5 Oct 2026 05:46:15 +0000 Subject: [PATCH 4/6] style(mcp): probe signature edits through from_ref Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/tracedecay-mcp/src/handlers/edit_check.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/edit_check.rs b/crates/tracedecay-mcp/src/handlers/edit_check.rs index 29e3bbf185..7def148940 100644 --- a/crates/tracedecay-mcp/src/handlers/edit_check.rs +++ b/crates/tracedecay-mcp/src/handlers/edit_check.rs @@ -335,8 +335,13 @@ mod tests { file: "caller.rs".to_owned(), line: 0, }; - let error = signature_edits(root.path(), &[target.clone()], &[target, caller], &[]) - .unwrap_err(); + let error = signature_edits( + root.path(), + std::slice::from_ref(&target), + &[target.clone(), caller], + &[], + ) + .unwrap_err(); assert!( matches!(error, TraceDecayError::Io(ref error) if error.kind() == std::io::ErrorKind::NotFound), "{missing}: {error}" From 3269b18f73f98578107d2afad45290a0a80f9581 Mon Sep 17 00:00:00 2001 From: "zackary.l.jackson" Date: Mon, 5 Oct 2026 06:34:05 +0000 Subject: [PATCH 5/6] fix(bench): restore bench targets after upstream merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/tracedecay/benches/coverage/admin.rs | 5 +++-- crates/tracedecay/benches/coverage/mod.rs | 3 ++- crates/tracedecay/benches/queries.rs | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/benches/coverage/admin.rs b/crates/tracedecay/benches/coverage/admin.rs index 4548b204d3..d1713195ed 100644 --- a/crates/tracedecay/benches/coverage/admin.rs +++ b/crates/tracedecay/benches/coverage/admin.rs @@ -1046,7 +1046,7 @@ pub(crate) fn groups(ctx: &QueryContext, out: &mut Vec) { args["include_storage_health"] = json!(true); } if tool == "tracedecay_configuration_observed_state" { - Query::prepared_read(label, tool, args, super::configuration_read_prime) + Query::prepared_read(label, tool, args, i, super::configuration_read_prime) } else { rq(tool, label, args) } @@ -1084,11 +1084,12 @@ pub(crate) fn groups(ctx: &QueryContext, out: &mut Vec) { } out.push(ToolGroup { tool: "tracedecay_configuration_get", - queries: five(|_i| { + queries: five(|i| { Query::prepared_read( "config_get", "tracedecay_configuration_get", json!({"key": ctx.seeds.config_key.clone().unwrap_or_else(|| "diagnostics.prewarm.v1".into())}), + i, super::configuration_read_prime, ) }), diff --git a/crates/tracedecay/benches/coverage/mod.rs b/crates/tracedecay/benches/coverage/mod.rs index 014afd058f..a15602f7a9 100644 --- a/crates/tracedecay/benches/coverage/mod.rs +++ b/crates/tracedecay/benches/coverage/mod.rs @@ -13,7 +13,8 @@ mod work; pub(crate) use admin_fixture::verify_admin_fixture; pub(crate) use admin::{ - verify_context_scout_fixture, verify_github_stack_signal_fixture, verify_native_fixture, + finish_context_scout_fixture, verify_context_scout_fixture, verify_github_stack_signal_fixture, + verify_native_fixture, }; #[cfg(unix)] pub(crate) use code::prepare_source_reconciliation; diff --git a/crates/tracedecay/benches/queries.rs b/crates/tracedecay/benches/queries.rs index 558311a9de..fc812b0cca 100644 --- a/crates/tracedecay/benches/queries.rs +++ b/crates/tracedecay/benches/queries.rs @@ -672,7 +672,10 @@ pub fn build_queries(ctx: &QueryContext) -> Vec { ) }), }); - if crate::repos::small_fixture_enabled() { + // `crate::repos` only exists inside the `large_repos` target; this file is + // also its own `[[bench]]` target, so the flag is read directly (same + // `TRACEDECAY_BENCH_SMALL_FIXTURE` authority). + if std::env::var_os("TRACEDECAY_BENCH_SMALL_FIXTURE").is_some() { groups.push(ToolGroup { tool: "tracedecay_callees", queries: vec![Query::prepared_read( From 1520866058aa4ce81bbfbb4204ee925c73d1ad78 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 5 Oct 2026 04:59:14 -0700 Subject: [PATCH 6/6] style(hooks): remove narrated test setup --- crates/tracedecay-hooks/src/delivery_spool.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/tracedecay-hooks/src/delivery_spool.rs b/crates/tracedecay-hooks/src/delivery_spool.rs index 6a416f349b..f5a0db60ff 100644 --- a/crates/tracedecay-hooks/src/delivery_spool.rs +++ b/crates/tracedecay-hooks/src/delivery_spool.rs @@ -821,8 +821,6 @@ mod tests { HookDeliveryReceiptSpoolV1::open(&root.0, Duration::ZERO).unwrap_err(), HookDeliverySpoolError::Busy ); - // A spent budget still takes the free staging lease; only the held - // publish lock is out of reach, so the receipt stages. let writer = HookDeliveryReceiptWriterV1::open_within(&root.0, Duration::ZERO).unwrap(); assert_eq!( writer.retain(&receipt()).unwrap(),